homelab-runtime-publisher
31 lines
13 KiB
JSON
31 lines
13 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"channel": "homelab-runtime",
|
|
"command_id": "BACKUP-FAILED-UNITS-ROOTCAUSE-214",
|
|
"status": "OK",
|
|
"rc": 0,
|
|
"host": "pve01",
|
|
"mode": "read-only",
|
|
"component": "backup-audit",
|
|
"started_at_utc": "2026-07-22T15:19:41Z",
|
|
"finished_at_utc": "2026-07-22T15:19:41Z",
|
|
"reference_register_checked": true,
|
|
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
|
|
"error_register_checked": true,
|
|
"error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
|
|
"command_sha256": "19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1",
|
|
"duplicate_failed_command_blocked": false,
|
|
"block_reason": null,
|
|
"execution_started": true,
|
|
"changes_made": false,
|
|
"sanitized": true,
|
|
"secrets_included": false,
|
|
"private_addresses_included": false,
|
|
"raw_evidence_retained_locally": true,
|
|
"raw_evidence_sha256": "342cecee34231056a87789ec2411b4a4e54071817393a6b227a5a74f2e6619c3",
|
|
"sanitized_output_sha256": "d51060b6b35d2cbb1667bc9bee8d6bc8dc2079c25fb2dcf949eda14fa37537cc",
|
|
"output_truncated_in_json": false,
|
|
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
|
"output": "PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212\nMISSING_RESULT_213=CONFIRMED\n===== homelab-backup-matrix-refresh =====\nResult=exit-code\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n===== homelab-vm160-remote-backup =====\nResult=exit-code\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nhomelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nVM160_TARGET_FINALIZE_STATUS=OK\nVM160_TARGET_PROMOTE_STATUS=OK\nVM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z\nARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst\nSIZE_BYTES=84995216465\nSHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72\nhomelab-vm160-remote-backup.service: Deactivated successfully.\nFinished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nhomelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\n===== homelab-cluster-admin-pull =====\nResult=exit-code\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nSTATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\n===== MATRIX_SYNTAX =====\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `('\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path(\"/etc/homelab-backup-matrix/matrix.json\")'\nMATRIX_SYNTAX_RC=2\n"
|
|
}
|