diff --git a/runtime/history/BACKUP-FAILED-UNITS-ROOTCAUSE-213.json b/runtime/history/BACKUP-FAILED-UNITS-ROOTCAUSE-213.json new file mode 100644 index 0000000..f51441f --- /dev/null +++ b/runtime/history/BACKUP-FAILED-UNITS-ROOTCAUSE-213.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-FAILED-UNITS-ROOTCAUSE-213", + "status": "FAIL", + "rc": 2, + "host": "pve01", + "mode": "read-only", + "component": "backup-audit", + "started_at_utc": "2026-07-22T15:16:25Z", + "finished_at_utc": "2026-07-22T15:16:25Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "fb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "5aa2486758a52989308a01aa788b4cd825645ba54a4ca5bc7ac0d46880d624ac", + "sanitized_output_sha256": "eb0649d407eb1c9f6ef0ac82df9b958a46ebf6fd91680b20839817454f33891a", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212\n× homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state\n Loaded: loaded (/etc/systemd/system/homelab-backup-matrix-refresh.service; static)\n Active: failed (Result: exit-code) since Wed 2026-07-22 10:11:15 MSK; 8h ago\n Invocation: 8a8289d574e745078d5316652c3a62cc\n Main PID: 4192756 (code=exited, status=1/FAILURE)\n Mem peak: 12.9M\n CPU: 582ms\n\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 22 10:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 22 10:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 22 10:11:15 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n\n× homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03\n Loaded: loaded (/etc/systemd/system/homelab-vm160-remote-backup.service; static)\n Active: failed (Result: exit-code) since Tue 2026-07-21 02:39:14 MSK; 1 day 15h ago\n Invocation: 603319cf8a194ed48ec2cca37b86f473\n Main PID: 2028253 (code=exited, status=1/FAILURE)\n Mem peak: 4.2M\n CPU: 97ms\n\nJul 21 02:39:12 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 21 02:39:14 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\n\n× homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01\n Loaded: loaded (/etc/systemd/system/homelab-cluster-admin-pull.service; static)\n Active: failed (Result: exit-code) since Tue 2026-07-21 02:37:04 MSK; 1 day 15h ago\n Invocation: 1f07cb8016434bdb92ac64ed740b7f3e\n Main PID: 2014947 (code=exited, status=1/FAILURE)\n Mem peak: 4.6M\n CPU: 64ms\n\nJul 21 02:36:57 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015088]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015089]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015090]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 02:37:04 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_SYNTAX_CHECK\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `('\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path(\"/etc/homelab-backup-matrix/matrix.json\")'\n" +} diff --git a/runtime/history/BACKUP-FAILED-UNITS-ROOTCAUSE-213.txt b/runtime/history/BACKUP-FAILED-UNITS-ROOTCAUSE-213.txt new file mode 100644 index 0000000..65289c6 --- /dev/null +++ b/runtime/history/BACKUP-FAILED-UNITS-ROOTCAUSE-213.txt @@ -0,0 +1,255 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-FAILED-UNITS-ROOTCAUSE-213 +STATUS=FAIL +RC=2 +HOST=pve01 +MODE=read-only +COMPONENT=backup-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=fb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=5aa2486758a52989308a01aa788b4cd825645ba54a4ca5bc7ac0d46880d624ac +SANITIZED_OUTPUT_SHA256=eb0649d407eb1c9f6ef0ac82df9b958a46ebf6fd91680b20839817454f33891a +OUTPUT_BEGIN +PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212 +× homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state + Loaded: loaded (/etc/systemd/system/homelab-backup-matrix-refresh.service; static) + Active: failed (Result: exit-code) since Wed 2026-07-22 10:11:15 MSK; 8h ago + Invocation: 8a8289d574e745078d5316652c3a62cc + Main PID: 4192756 (code=exited, status=1/FAILURE) + Mem peak: 12.9M + CPU: 582ms + +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41 +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30 +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6 +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0 +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5 +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +Jul 22 10:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +Jul 22 10:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Jul 22 10:11:15 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. + +× homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03 + Loaded: loaded (/etc/systemd/system/homelab-vm160-remote-backup.service; static) + Active: failed (Result: exit-code) since Tue 2026-07-21 02:39:14 MSK; 1 day 15h ago + Invocation: 603319cf8a194ed48ec2cca37b86f473 + Main PID: 2028253 (code=exited, status=1/FAILURE) + Mem peak: 4.2M + CPU: 97ms + +Jul 21 02:39:12 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +Jul 21 02:39:14 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. + +× homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01 + Loaded: loaded (/etc/systemd/system/homelab-cluster-admin-pull.service; static) + Active: failed (Result: exit-code) since Tue 2026-07-21 02:37:04 MSK; 1 day 15h ago + Invocation: 1f07cb8016434bdb92ac64ed740b7f3e + Main PID: 2014947 (code=exited, status=1/FAILURE) + Mem peak: 4.6M + CPU: 64ms + +Jul 21 02:36:57 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... +Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015088]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt +Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015089]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md +Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015090]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer +Jul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE +Jul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. +Jul 21 02:37:04 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_SYNTAX_CHECK +/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `(' +/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path("/etc/homelab-backup-matrix/matrix.json")' + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index c17a06c..f51441f 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-OFFHOST-DELTA-212", - "status": "OK", - "rc": 0, + "command_id": "BACKUP-FAILED-UNITS-ROOTCAUSE-213", + "status": "FAIL", + "rc": 2, "host": "pve01", "mode": "read-only", - "component": "cluster-audit", - "started_at_utc": "2026-07-22T15:00:05Z", - "finished_at_utc": "2026-07-22T15:00:05Z", + "component": "backup-audit", + "started_at_utc": "2026-07-22T15:16:25Z", + "finished_at_utc": "2026-07-22T15:16:25Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "cd7bfba0a3e05e136f85c1362ba4b34c9e142ef49f4621874743aebf2b665eee", + "command_sha256": "fb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682", - "sanitized_output_sha256": "10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682", + "raw_evidence_sha256": "5aa2486758a52989308a01aa788b4cd825645ba54a4ca5bc7ac0d46880d624ac", + "sanitized_output_sha256": "eb0649d407eb1c9f6ef0ac82df9b958a46ebf6fd91680b20839817454f33891a", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "validation=PASS\n=== TARGET_UNITS ===\nResult=exit-code\nExecMainStatus=1\nId=homelab-backup-matrix-refresh.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\n\nResult=exit-code\nExecMainStatus=1\nId=homelab-vm160-remote-backup.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\n\nResult=exit-code\nExecMainStatus=1\nId=homelab-cluster-admin-pull.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\n=== TARGET_FAILURES ===\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nhomelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01\nhomelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\n=== HEALTH_MARKERS ===\nFILE=/var/lib/homelab-health/backup-framework.txt MTIME=2026-07-22 10:11:15.588809944 +0300 SIZE=803\nSTATUS=OK\nTS=2026-07-22T07:11:15Z\nTYPE=backup-matrix-generated\nFILE=/var/lib/homelab-health/backup-sla.txt MTIME=2026-07-22 17:45:04.576070554 +0300 SIZE=154\nSTATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nFILE=/var/lib/homelab-health/backup-coverage-matrix.txt MTIME=2026-07-22 10:11:15.236509358 +0300 SIZE=808\nSTATUS=OK\nTS=2026-07-22T07:11:15Z\nTYPE=backup-matrix-generated\nFILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt MTIME=2026-07-22 10:11:15.694007276 +0300 SIZE=800\nSTATUS=OK\nTS=2026-07-22T07:11:15Z\nTYPE=backup-matrix-generated\nFILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt MTIME=2026-07-09 08:07:47.213452046 +0300 SIZE=217\nSTATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\nFILE=/var/lib/homelab-health/vm-backup.txt MTIME=2026-07-22 07:35:14.956502473 +0300 SIZE=170\nSTATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active\n=== LOCAL_ARCHIVES ===\nARCHIVE=vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400\nARCHIVE=vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720\nARCHIVE=vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330\nARCHIVE=vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500\nARCHIVE=vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730\nARCHIVE=vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440\nARCHIVE=vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290\nARCHIVE=vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120\nARCHIVE=vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020\nARCHIVE=vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470\nARCHIVE=vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950\nARCHIVE=vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120\n=== RECENT_ERRORS ===\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nCHANGES_MADE=NO\nREFERENCE_CHECK=OK\nERROR_REGISTER_CHECK=OK\nRESULT=OK\nBACKUP_OFFHOST_DELTA=COMPLETE\n" + "output": "PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212\n× homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state\n Loaded: loaded (/etc/systemd/system/homelab-backup-matrix-refresh.service; static)\n Active: failed (Result: exit-code) since Wed 2026-07-22 10:11:15 MSK; 8h ago\n Invocation: 8a8289d574e745078d5316652c3a62cc\n Main PID: 4192756 (code=exited, status=1/FAILURE)\n Mem peak: 12.9M\n CPU: 582ms\n\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 22 10:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 22 10:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 22 10:11:15 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n\n× homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03\n Loaded: loaded (/etc/systemd/system/homelab-vm160-remote-backup.service; static)\n Active: failed (Result: exit-code) since Tue 2026-07-21 02:39:14 MSK; 1 day 15h ago\n Invocation: 603319cf8a194ed48ec2cca37b86f473\n Main PID: 2028253 (code=exited, status=1/FAILURE)\n Mem peak: 4.2M\n CPU: 97ms\n\nJul 21 02:39:12 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 21 02:39:14 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\n\n× homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01\n Loaded: loaded (/etc/systemd/system/homelab-cluster-admin-pull.service; static)\n Active: failed (Result: exit-code) since Tue 2026-07-21 02:37:04 MSK; 1 day 15h ago\n Invocation: 1f07cb8016434bdb92ac64ed740b7f3e\n Main PID: 2014947 (code=exited, status=1/FAILURE)\n Mem peak: 4.6M\n CPU: 64ms\n\nJul 21 02:36:57 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015088]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015089]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015090]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 02:37:04 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_SYNTAX_CHECK\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `('\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path(\"/etc/homelab-backup-matrix/matrix.json\")'\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index b1ecf49..65289c6 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,123 +1,255 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-OFFHOST-DELTA-212 -STATUS=OK -RC=0 +COMMAND_ID=BACKUP-FAILED-UNITS-ROOTCAUSE-213 +STATUS=FAIL +RC=2 HOST=pve01 MODE=read-only -COMPONENT=cluster-audit +COMPONENT=backup-audit REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=cd7bfba0a3e05e136f85c1362ba4b34c9e142ef49f4621874743aebf2b665eee +COMMAND_SHA256=fb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682 -SANITIZED_OUTPUT_SHA256=10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682 +RAW_EVIDENCE_SHA256=5aa2486758a52989308a01aa788b4cd825645ba54a4ca5bc7ac0d46880d624ac +SANITIZED_OUTPUT_SHA256=eb0649d407eb1c9f6ef0ac82df9b958a46ebf6fd91680b20839817454f33891a OUTPUT_BEGIN -validation=PASS -=== TARGET_UNITS === -Result=exit-code -ExecMainStatus=1 -Id=homelab-backup-matrix-refresh.service -LoadState=loaded -ActiveState=failed -SubState=failed +PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212 +× homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state + Loaded: loaded (/etc/systemd/system/homelab-backup-matrix-refresh.service; static) + Active: failed (Result: exit-code) since Wed 2026-07-22 10:11:15 MSK; 8h ago + Invocation: 8a8289d574e745078d5316652c3a62cc + Main PID: 4192756 (code=exited, status=1/FAILURE) + Mem peak: 12.9M + CPU: 582ms -Result=exit-code -ExecMainStatus=1 -Id=homelab-vm160-remote-backup.service -LoadState=loaded -ActiveState=failed -SubState=failed +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41 +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30 +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6 +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0 +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5 +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +Jul 22 10:11:15 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +Jul 22 10:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +Jul 22 10:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Jul 22 10:11:15 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Result=exit-code -ExecMainStatus=1 -Id=homelab-cluster-admin-pull.service -LoadState=loaded -ActiveState=failed -SubState=failed -=== TARGET_FAILURES === -homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -homelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01 -homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -=== HEALTH_MARKERS === -FILE=/var/lib/homelab-health/backup-framework.txt MTIME=2026-07-22 10:11:15.588809944 +0300 SIZE=803 -STATUS=OK -TS=2026-07-22T07:11:15Z -TYPE=backup-matrix-generated -FILE=/var/lib/homelab-health/backup-sla.txt MTIME=2026-07-22 17:45:04.576070554 +0300 SIZE=154 -STATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -FILE=/var/lib/homelab-health/backup-coverage-matrix.txt MTIME=2026-07-22 10:11:15.236509358 +0300 SIZE=808 -STATUS=OK -TS=2026-07-22T07:11:15Z -TYPE=backup-matrix-generated -FILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt MTIME=2026-07-22 10:11:15.694007276 +0300 SIZE=800 -STATUS=OK -TS=2026-07-22T07:11:15Z -TYPE=backup-matrix-generated -FILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt MTIME=2026-07-09 08:07:47.213452046 +0300 SIZE=217 -STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md -FILE=/var/lib/homelab-health/vm-backup.txt MTIME=2026-07-22 07:35:14.956502473 +0300 SIZE=170 -STATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active -=== LOCAL_ARCHIVES === -ARCHIVE=vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400 -ARCHIVE=vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720 -ARCHIVE=vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330 -ARCHIVE=vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500 -ARCHIVE=vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730 -ARCHIVE=vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440 -ARCHIVE=vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290 -ARCHIVE=vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120 -ARCHIVE=vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020 -ARCHIVE=vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470 -ARCHIVE=vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950 -ARCHIVE=vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120 -=== RECENT_ERRORS === -homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +× homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03 + Loaded: loaded (/etc/systemd/system/homelab-vm160-remote-backup.service; static) + Active: failed (Result: exit-code) since Tue 2026-07-21 02:39:14 MSK; 1 day 15h ago + Invocation: 603319cf8a194ed48ec2cca37b86f473 + Main PID: 2028253 (code=exited, status=1/FAILURE) + Mem peak: 4.2M + CPU: 97ms + +Jul 21 02:39:12 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... +Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE +Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. +Jul 21 02:39:14 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. + +× homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01 + Loaded: loaded (/etc/systemd/system/homelab-cluster-admin-pull.service; static) + Active: failed (Result: exit-code) since Tue 2026-07-21 02:37:04 MSK; 1 day 15h ago + Invocation: 1f07cb8016434bdb92ac64ed740b7f3e + Main PID: 2014947 (code=exited, status=1/FAILURE) + Mem peak: 4.6M + CPU: 64ms + +Jul 21 02:36:57 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... +Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015088]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt +Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015089]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md +Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015090]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer +Jul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE +Jul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. +Jul 21 02:37:04 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -CHANGES_MADE=NO -REFERENCE_CHECK=OK -ERROR_REGISTER_CHECK=OK -RESULT=OK -BACKUP_OFFHOST_DELTA=COMPLETE +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... +MATRIX_STATE_STATUS=FAIL +MATRIX_STATE_JOBS_TOTAL=41 +MATRIX_STATE_JOBS_OK=30 +MATRIX_STATE_JOBS_BAD=6 +MATRIX_STATE_JOBS_UNMAPPED=0 +MATRIX_STATE_JOBS_DISABLED=5 +MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json +MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_SYNTAX_CHECK +/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `(' +/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path("/etc/homelab-backup-matrix/matrix.json")' OUTPUT_END CHAT_OUTPUT_END