diff --git a/runtime/history/RUNTIME-PUBLISH-GAP-DIAG-216.json b/runtime/history/RUNTIME-PUBLISH-GAP-DIAG-216.json new file mode 100644 index 0000000..58afa8f --- /dev/null +++ b/runtime/history/RUNTIME-PUBLISH-GAP-DIAG-216.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "RUNTIME-PUBLISH-GAP-DIAG-216", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-22T15:27:36Z", + "finished_at_utc": "2026-07-22T15:27:36Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "7e59c8be0d1806c75e6a70e6272d23228e685ed0f7ce29ff23655e5cb6cab201", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0", + "sanitized_output_sha256": "d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "PREVIOUS_CONFIRMED=BACKUP-OFFHOST-DELTA-212\n=== LEDGER 213-215 ===\n2026-07-22T15:16:25Z\tBACKUP-FAILED-UNITS-ROOTCAUSE-213\tfb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435\t2\tFAIL\tbackup-audit\n2026-07-22T15:19:41Z\tBACKUP-FAILED-UNITS-ROOTCAUSE-214\t19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1\t0\tOK\tbackup-audit\n=== LOCAL FILES 213-215 ===\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt\n/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.log\n/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.log\n=== RECENT PUBLIC RESULTS ===\n1784733581.6637746080 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt\n1784733581.6615274220 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json\n1784733581.6428190850 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt\n1784733385.6042597560 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt\n1784733385.6012319060 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json\n1784733385.5822310630 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt\n1784732405.9184316600 BACKUP-OFFHOST-DELTA-212-20260722T150005Z-latest.txt\n1784732405.9148605780 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.json\n1784732405.8929104080 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.txt\n1784732035.7581781390 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z-latest.txt\n1784732035.7560127120 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.json\n1784732035.7377798780 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.txt\n1784730859.9011235460 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z-latest.txt\n1784730859.8981025540 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.json\n1784730859.8809036410 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.txt\n1784730606.0033666560 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z-latest.txt\n1784730606.0015773800 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.json\n1784730605.9797819180 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.txt\n1784730456.7690908780 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z-latest.txt\n1784730456.7669407010 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z.json\nCHANGES_MADE=NO\n" +} diff --git a/runtime/history/RUNTIME-PUBLISH-GAP-DIAG-216.txt b/runtime/history/RUNTIME-PUBLISH-GAP-DIAG-216.txt new file mode 100644 index 0000000..0b1e6f3 --- /dev/null +++ b/runtime/history/RUNTIME-PUBLISH-GAP-DIAG-216.txt @@ -0,0 +1,59 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=RUNTIME-PUBLISH-GAP-DIAG-216 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=7e59c8be0d1806c75e6a70e6272d23228e685ed0f7ce29ff23655e5cb6cab201 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0 +SANITIZED_OUTPUT_SHA256=d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0 +OUTPUT_BEGIN +PREVIOUS_CONFIRMED=BACKUP-OFFHOST-DELTA-212 +=== LEDGER 213-215 === +2026-07-22T15:16:25Z BACKUP-FAILED-UNITS-ROOTCAUSE-213 fb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435 2 FAIL backup-audit +2026-07-22T15:19:41Z BACKUP-FAILED-UNITS-ROOTCAUSE-214 19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1 0 OK backup-audit +=== LOCAL FILES 213-215 === +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt +/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.log +/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.log +=== RECENT PUBLIC RESULTS === +1784733581.6637746080 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt +1784733581.6615274220 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json +1784733581.6428190850 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt +1784733385.6042597560 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt +1784733385.6012319060 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json +1784733385.5822310630 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt +1784732405.9184316600 BACKUP-OFFHOST-DELTA-212-20260722T150005Z-latest.txt +1784732405.9148605780 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.json +1784732405.8929104080 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.txt +1784732035.7581781390 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z-latest.txt +1784732035.7560127120 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.json +1784732035.7377798780 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.txt +1784730859.9011235460 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z-latest.txt +1784730859.8981025540 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.json +1784730859.8809036410 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.txt +1784730606.0033666560 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z-latest.txt +1784730606.0015773800 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.json +1784730605.9797819180 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.txt +1784730456.7690908780 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z-latest.txt +1784730456.7669407010 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z.json +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index cf8352e..58afa8f 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-FAILED-UNITS-ROOTCAUSE-214", + "command_id": "RUNTIME-PUBLISH-GAP-DIAG-216", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "backup-audit", - "started_at_utc": "2026-07-22T15:19:41Z", - "finished_at_utc": "2026-07-22T15:19:41Z", + "component": "cluster-audit", + "started_at_utc": "2026-07-22T15:27:36Z", + "finished_at_utc": "2026-07-22T15:27:36Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1", + "command_sha256": "7e59c8be0d1806c75e6a70e6272d23228e685ed0f7ce29ff23655e5cb6cab201", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "342cecee34231056a87789ec2411b4a4e54071817393a6b227a5a74f2e6619c3", - "sanitized_output_sha256": "d51060b6b35d2cbb1667bc9bee8d6bc8dc2079c25fb2dcf949eda14fa37537cc", + "raw_evidence_sha256": "d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0", + "sanitized_output_sha256": "d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212\nMISSING_RESULT_213=CONFIRMED\n===== homelab-backup-matrix-refresh =====\nResult=exit-code\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nStarting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nMATRIX_STATE_STATUS=FAIL\nMATRIX_STATE_JOBS_TOTAL=41\nMATRIX_STATE_JOBS_OK=30\nMATRIX_STATE_JOBS_BAD=6\nMATRIX_STATE_JOBS_UNMAPPED=0\nMATRIX_STATE_JOBS_DISABLED=5\nMATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nMATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n===== homelab-vm160-remote-backup =====\nResult=exit-code\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nhomelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nVM160_TARGET_FINALIZE_STATUS=OK\nVM160_TARGET_PROMOTE_STATUS=OK\nVM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z\nARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst\nSIZE_BYTES=84995216465\nSHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72\nhomelab-vm160-remote-backup.service: Deactivated successfully.\nFinished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nhomelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nStarting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nhomelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nFailed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\n===== homelab-cluster-admin-pull =====\nResult=exit-code\nExecMainStatus=1\nActiveState=failed\nSubState=failed\nSTATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nStarting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nSTATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nSTATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nSTATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nhomelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nFailed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\n===== MATRIX_SYNTAX =====\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `('\n/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path(\"/etc/homelab-backup-matrix/matrix.json\")'\nMATRIX_SYNTAX_RC=2\n" + "output": "PREVIOUS_CONFIRMED=BACKUP-OFFHOST-DELTA-212\n=== LEDGER 213-215 ===\n2026-07-22T15:16:25Z\tBACKUP-FAILED-UNITS-ROOTCAUSE-213\tfb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435\t2\tFAIL\tbackup-audit\n2026-07-22T15:19:41Z\tBACKUP-FAILED-UNITS-ROOTCAUSE-214\t19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1\t0\tOK\tbackup-audit\n=== LOCAL FILES 213-215 ===\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt\n/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.log\n/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.log\n=== RECENT PUBLIC RESULTS ===\n1784733581.6637746080 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt\n1784733581.6615274220 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json\n1784733581.6428190850 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt\n1784733385.6042597560 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt\n1784733385.6012319060 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json\n1784733385.5822310630 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt\n1784732405.9184316600 BACKUP-OFFHOST-DELTA-212-20260722T150005Z-latest.txt\n1784732405.9148605780 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.json\n1784732405.8929104080 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.txt\n1784732035.7581781390 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z-latest.txt\n1784732035.7560127120 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.json\n1784732035.7377798780 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.txt\n1784730859.9011235460 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z-latest.txt\n1784730859.8981025540 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.json\n1784730859.8809036410 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.txt\n1784730606.0033666560 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z-latest.txt\n1784730606.0015773800 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.json\n1784730605.9797819180 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.txt\n1784730456.7690908780 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z-latest.txt\n1784730456.7669407010 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z.json\nCHANGES_MADE=NO\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 1e0d519..0b1e6f3 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,165 +1,59 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-FAILED-UNITS-ROOTCAUSE-214 +COMMAND_ID=RUNTIME-PUBLISH-GAP-DIAG-216 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=backup-audit +COMPONENT=cluster-audit REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1 +COMMAND_SHA256=7e59c8be0d1806c75e6a70e6272d23228e685ed0f7ce29ff23655e5cb6cab201 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=342cecee34231056a87789ec2411b4a4e54071817393a6b227a5a74f2e6619c3 -SANITIZED_OUTPUT_SHA256=d51060b6b35d2cbb1667bc9bee8d6bc8dc2079c25fb2dcf949eda14fa37537cc +RAW_EVIDENCE_SHA256=d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0 +SANITIZED_OUTPUT_SHA256=d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0 OUTPUT_BEGIN -PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212 -MISSING_RESULT_213=CONFIRMED -===== homelab-backup-matrix-refresh ===== -Result=exit-code -ExecMainStatus=1 -ActiveState=failed -SubState=failed -MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -MATRIX_STATE_STATUS=FAIL -MATRIX_STATE_JOBS_TOTAL=41 -MATRIX_STATE_JOBS_OK=30 -MATRIX_STATE_JOBS_BAD=6 -MATRIX_STATE_JOBS_UNMAPPED=0 -MATRIX_STATE_JOBS_DISABLED=5 -MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -MATRIX_STATE_STATUS=FAIL -MATRIX_STATE_JOBS_TOTAL=41 -MATRIX_STATE_JOBS_OK=30 -MATRIX_STATE_JOBS_BAD=6 -MATRIX_STATE_JOBS_UNMAPPED=0 -MATRIX_STATE_JOBS_DISABLED=5 -MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -MATRIX_STATE_STATUS=FAIL -MATRIX_STATE_JOBS_TOTAL=41 -MATRIX_STATE_JOBS_OK=30 -MATRIX_STATE_JOBS_BAD=6 -MATRIX_STATE_JOBS_UNMAPPED=0 -MATRIX_STATE_JOBS_DISABLED=5 -MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -===== homelab-vm160-remote-backup ===== -Result=exit-code -ExecMainStatus=1 -ActiveState=failed -SubState=failed -homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak. -Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -VM160_TARGET_FINALIZE_STATUS=OK -VM160_TARGET_PROMOTE_STATUS=OK -VM160_REMOTE_BACKUP_STATUS=OK -TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z -ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst -SIZE_BYTES=84995216465 -SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72 -homelab-vm160-remote-backup.service: Deactivated successfully. -Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak. -Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -===== homelab-cluster-admin-pull ===== -Result=exit-code -ExecMainStatus=1 -ActiveState=failed -SubState=failed -STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -===== MATRIX_SYNTAX ===== -/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `(' -/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path("/etc/homelab-backup-matrix/matrix.json")' -MATRIX_SYNTAX_RC=2 +PREVIOUS_CONFIRMED=BACKUP-OFFHOST-DELTA-212 +=== LEDGER 213-215 === +2026-07-22T15:16:25Z BACKUP-FAILED-UNITS-ROOTCAUSE-213 fb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435 2 FAIL backup-audit +2026-07-22T15:19:41Z BACKUP-FAILED-UNITS-ROOTCAUSE-214 19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1 0 OK backup-audit +=== LOCAL FILES 213-215 === +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt +/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt +/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.log +/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.log +=== RECENT PUBLIC RESULTS === +1784733581.6637746080 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt +1784733581.6615274220 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json +1784733581.6428190850 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt +1784733385.6042597560 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt +1784733385.6012319060 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json +1784733385.5822310630 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt +1784732405.9184316600 BACKUP-OFFHOST-DELTA-212-20260722T150005Z-latest.txt +1784732405.9148605780 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.json +1784732405.8929104080 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.txt +1784732035.7581781390 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z-latest.txt +1784732035.7560127120 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.json +1784732035.7377798780 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.txt +1784730859.9011235460 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z-latest.txt +1784730859.8981025540 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.json +1784730859.8809036410 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.txt +1784730606.0033666560 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z-latest.txt +1784730606.0015773800 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.json +1784730605.9797819180 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.txt +1784730456.7690908780 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z-latest.txt +1784730456.7669407010 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z.json +CHANGES_MADE=NO OUTPUT_END CHAT_OUTPUT_END