{
  "schema_version": 1,
  "channel": "homelab-runtime",
  "command_id": "BACKUP-FAILED-UNITS-ROOTCAUSE-213",
  "status": "FAIL",
  "rc": 2,
  "host": "pve01",
  "mode": "read-only",
  "component": "backup-audit",
  "started_at_utc": "2026-07-22T15:16:25Z",
  "finished_at_utc": "2026-07-22T15:16:25Z",
  "reference_register_checked": true,
  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
  "error_register_checked": true,
  "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
  "command_sha256": "fb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435",
  "duplicate_failed_command_blocked": false,
  "block_reason": null,
  "execution_started": true,
  "changes_made": false,
  "sanitized": true,
  "secrets_included": false,
  "private_addresses_included": false,
  "raw_evidence_retained_locally": true,
  "raw_evidence_sha256": "5aa2486758a52989308a01aa788b4cd825645ba54a4ca5bc7ac0d46880d624ac",
  "sanitized_output_sha256": "eb0649d407eb1c9f6ef0ac82df9b958a46ebf6fd91680b20839817454f33891a",
  "output_truncated_in_json": false,
  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
  "output": "PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212\n× homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state\n     Loaded: loaded (/etc/systemd/system/homelab-backup-matrix-refresh.service; static)\n     Active: failed (Result: exit-code) since Wed 2026-07-22 10:11:15 MSK; 8h ago\n Invocation: 8a8289d574e745078d5316652c3a62cc\n   Main PID: 4192756 (code=exited, status=1/FAILURE)\n   Mem peak: 12.9M\n        CPU: 582ms\n\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 22 10:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 22 10:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 22 10:11:15 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n\n× homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03\n     Loaded: loaded (/etc/systemd/system/homelab-vm160-remote-backup.service; static)\n     Active: failed (Result: exit-code) since Tue 2026-07-21 02:39:14 MSK; 1 day 15h ago\n Invocation: 603319cf8a194ed48ec2cca37b86f473\n   Main PID: 2028253 (code=exited, status=1/FAILURE)\n   Mem peak: 4.2M\n        CPU: 97ms\n\nJul 21 02:39:12 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 21 02:39:14 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\n\n× homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01\n     Loaded: loaded (/etc/systemd/system/homelab-cluster-admin-pull.service; static)\n     Active: failed (Result: exit-code) since Tue 2026-07-21 02:37:04 MSK; 1 day 15h ago\n Invocation: 1f07cb8016434bdb92ac64ed740b7f3e\n   Main PID: 2014947 (code=exited, status=1/FAILURE)\n   Mem peak: 4.6M\n        CPU: 64ms\n\nJul 21 02:36:57 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015088]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015089]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015090]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 02:37:04 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_SYNTAX_CHECK\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `('\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path(\"/etc/homelab-backup-matrix/matrix.json\")'\n"
}