{
  "schema_version": 1,
  "channel": "homelab-runtime",
  "command_id": "BACKUP-FAILED-UNITS-ROOTCAUSE-214",
  "status": "OK",
  "rc": 0,
  "host": "pve01",
  "mode": "read-only",
  "component": "backup-audit",
  "started_at_utc": "2026-07-22T15:19:41Z",
  "finished_at_utc": "2026-07-22T15:19:41Z",
  "reference_register_checked": true,
  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
  "error_register_checked": true,
  "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
  "command_sha256": "19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1",
  "duplicate_failed_command_blocked": false,
  "block_reason": null,
  "execution_started": true,
  "changes_made": false,
  "sanitized": true,
  "secrets_included": false,
  "private_addresses_included": false,
  "raw_evidence_retained_locally": true,
  "raw_evidence_sha256": "342cecee34231056a87789ec2411b4a4e54071817393a6b227a5a74f2e6619c3",
  "sanitized_output_sha256": "d51060b6b35d2cbb1667bc9bee8d6bc8dc2079c25fb2dcf949eda14fa37537cc",
  "output_truncated_in_json": false,
  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
  "output": "PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212\nMISSING_RESULT_213=CONFIRMED\n===== homelab-backup-matrix-refresh =====\nResult=exit-code\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n===== homelab-vm160-remote-backup =====\nResult=exit-code\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nhomelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nVM160_TARGET_FINALIZE_STATUS=OK\nVM160_TARGET_PROMOTE_STATUS=OK\nVM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z\nARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst\nSIZE_BYTES=84995216465\nSHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72\nhomelab-vm160-remote-backup.service: Deactivated successfully.\nFinished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nhomelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\n===== homelab-cluster-admin-pull =====\nResult=exit-code\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nSTATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\n===== MATRIX_SYNTAX =====\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `('\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path(\"/etc/homelab-backup-matrix/matrix.json\")'\nMATRIX_SYNTAX_RC=2\n"
}