homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity

runtime: publish RUNTIME-PUBLISH-GAP-DIAG-216

Browse Source
This commit is contained in:
homelab-runtime-publisher
2026-07-22 15:27:37 +00:00
parent 48a6772b7d
commit 744480384c
4 changed files with 137 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/history/RUNTIME-PUBLISH-GAP-DIAG-216.json
+30
View File
@@ -0,0 +1,30 @@
{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "RUNTIME-PUBLISH-GAP-DIAG-216",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "read-only",
"component": "cluster-audit",
"started_at_utc": "2026-07-22T15:27:36Z",
"finished_at_utc": "2026-07-22T15:27:36Z",
"reference_register_checked": true,
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
"error_register_checked": true,
"error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
"command_sha256": "7e59c8be0d1806c75e6a70e6272d23228e685ed0f7ce29ff23655e5cb6cab201",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
"changes_made": false,
"sanitized": true,
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0",
"sanitized_output_sha256": "d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "PREVIOUS_CONFIRMED=BACKUP-OFFHOST-DELTA-212\n=== LEDGER 213-215 ===\n2026-07-22T15:16:25Z\tBACKUP-FAILED-UNITS-ROOTCAUSE-213\tfb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435\t2\tFAIL\tbackup-audit\n2026-07-22T15:19:41Z\tBACKUP-FAILED-UNITS-ROOTCAUSE-214\t19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1\t0\tOK\tbackup-audit\n=== LOCAL FILES 213-215 ===\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt\n/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt\n/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.log\n/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.log\n=== RECENT PUBLIC RESULTS ===\n1784733581.6637746080 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt\n1784733581.6615274220 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json\n1784733581.6428190850 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt\n1784733385.6042597560 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt\n1784733385.6012319060 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json\n1784733385.5822310630 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt\n1784732405.9184316600 BACKUP-OFFHOST-DELTA-212-20260722T150005Z-latest.txt\n1784732405.9148605780 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.json\n1784732405.8929104080 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.txt\n1784732035.7581781390 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z-latest.txt\n1784732035.7560127120 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.json\n1784732035.7377798780 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.txt\n1784730859.9011235460 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z-latest.txt\n1784730859.8981025540 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.json\n1784730859.8809036410 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.txt\n1784730606.0033666560 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z-latest.txt\n1784730606.0015773800 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.json\n1784730605.9797819180 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.txt\n1784730456.7690908780 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z-latest.txt\n1784730456.7669407010 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z.json\nCHANGES_MADE=NO\n"
}
runtime/history/RUNTIME-PUBLISH-GAP-DIAG-216.txt
+59
View File
@@ -0,0 +1,59 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=RUNTIME-PUBLISH-GAP-DIAG-216
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=7e59c8be0d1806c75e6a70e6272d23228e685ed0f7ce29ff23655e5cb6cab201
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0
SANITIZED_OUTPUT_SHA256=d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0
OUTPUT_BEGIN
PREVIOUS_CONFIRMED=BACKUP-OFFHOST-DELTA-212
=== LEDGER 213-215 ===
2026-07-22T15:16:25Z BACKUP-FAILED-UNITS-ROOTCAUSE-213 fb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435 2 FAIL backup-audit
2026-07-22T15:19:41Z BACKUP-FAILED-UNITS-ROOTCAUSE-214 19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1 0 OK backup-audit
=== LOCAL FILES 213-215 ===
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt
/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.log
/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.log
=== RECENT PUBLIC RESULTS ===
1784733581.6637746080 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt
1784733581.6615274220 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json
1784733581.6428190850 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt
1784733385.6042597560 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt
1784733385.6012319060 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json
1784733385.5822310630 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt
1784732405.9184316600 BACKUP-OFFHOST-DELTA-212-20260722T150005Z-latest.txt
1784732405.9148605780 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.json
1784732405.8929104080 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.txt
1784732035.7581781390 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z-latest.txt
1784732035.7560127120 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.json
1784732035.7377798780 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.txt
1784730859.9011235460 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z-latest.txt
1784730859.8981025540 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.json
1784730859.8809036410 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.txt
1784730606.0033666560 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z-latest.txt
1784730606.0015773800 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.json
1784730605.9797819180 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.txt
1784730456.7690908780 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z-latest.txt
1784730456.7669407010 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z.json
CHANGES_MADE=NO
OUTPUT_END
CHAT_OUTPUT_END
runtime/latest.json
+8 -8
View File
File diff suppressed because one or more lines are too long
runtime/latest.txt
+40 -146
View File
@@ -1,165 +1,59 @@
CHAT_OUTPUT_BEGIN
COMMAND_ID=BACKUP-FAILED-UNITS-ROOTCAUSE-214
COMMAND_ID=RUNTIME-PUBLISH-GAP-DIAG-216
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=backup-audit
COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1
COMMAND_SHA256=7e59c8be0d1806c75e6a70e6272d23228e685ed0f7ce29ff23655e5cb6cab201
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=342cecee34231056a87789ec2411b4a4e54071817393a6b227a5a74f2e6619c3
SANITIZED_OUTPUT_SHA256=d51060b6b35d2cbb1667bc9bee8d6bc8dc2079c25fb2dcf949eda14fa37537cc
RAW_EVIDENCE_SHA256=d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0
SANITIZED_OUTPUT_SHA256=d960ffb487a4fd038b366d566f09939731854b6a723f7c2255698cd5d5b758f0
OUTPUT_BEGIN
PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212
MISSING_RESULT_213=CONFIRMED
===== homelab-backup-matrix-refresh =====
Result=exit-code
ExecMainStatus=1
ActiveState=failed
SubState=failed
MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
MATRIX_STATE_STATUS=FAIL
MATRIX_STATE_JOBS_TOTAL=41
MATRIX_STATE_JOBS_OK=30
MATRIX_STATE_JOBS_BAD=6
MATRIX_STATE_JOBS_UNMAPPED=0
MATRIX_STATE_JOBS_DISABLED=5
MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
MATRIX_STATE_STATUS=FAIL
MATRIX_STATE_JOBS_TOTAL=41
MATRIX_STATE_JOBS_OK=30
MATRIX_STATE_JOBS_BAD=6
MATRIX_STATE_JOBS_UNMAPPED=0
MATRIX_STATE_JOBS_DISABLED=5
MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
MATRIX_STATE_STATUS=FAIL
MATRIX_STATE_JOBS_TOTAL=41
MATRIX_STATE_JOBS_OK=30
MATRIX_STATE_JOBS_BAD=6
MATRIX_STATE_JOBS_UNMAPPED=0
MATRIX_STATE_JOBS_DISABLED=5
MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
===== homelab-vm160-remote-backup =====
Result=exit-code
ExecMainStatus=1
ActiveState=failed
SubState=failed
homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak.
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
VM160_TARGET_FINALIZE_STATUS=OK
VM160_TARGET_PROMOTE_STATUS=OK
VM160_REMOTE_BACKUP_STATUS=OK
TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z
ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst
SIZE_BYTES=84995216465
SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72
homelab-vm160-remote-backup.service: Deactivated successfully.
Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak.
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
===== homelab-cluster-admin-pull =====
Result=exit-code
ExecMainStatus=1
ActiveState=failed
SubState=failed
STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
===== MATRIX_SYNTAX =====
/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `('
/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path("/etc/homelab-backup-matrix/matrix.json")'
MATRIX_SYNTAX_RC=2
PREVIOUS_CONFIRMED=BACKUP-OFFHOST-DELTA-212
=== LEDGER 213-215 ===
2026-07-22T15:16:25Z BACKUP-FAILED-UNITS-ROOTCAUSE-213 fb59f580fefab96bf42d4c9f720847f498158079695bebebca527795287c0435 2 FAIL backup-audit
2026-07-22T15:19:41Z BACKUP-FAILED-UNITS-ROOTCAUSE-214 19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1 0 OK backup-audit
=== LOCAL FILES 213-215 ===
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt
/var/lib/homelab-chat-run/public/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt
/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.log
/var/lib/homelab-chat-run/raw/BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.log
=== RECENT PUBLIC RESULTS ===
1784733581.6637746080 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z-latest.txt
1784733581.6615274220 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.json
1784733581.6428190850 BACKUP-FAILED-UNITS-ROOTCAUSE-214-20260722T151941Z.txt
1784733385.6042597560 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z-latest.txt
1784733385.6012319060 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.json
1784733385.5822310630 BACKUP-FAILED-UNITS-ROOTCAUSE-213-20260722T151625Z.txt
1784732405.9184316600 BACKUP-OFFHOST-DELTA-212-20260722T150005Z-latest.txt
1784732405.9148605780 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.json
1784732405.8929104080 BACKUP-OFFHOST-DELTA-212-20260722T150005Z.txt
1784732035.7581781390 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z-latest.txt
1784732035.7560127120 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.json
1784732035.7377798780 BACKUP-OFFHOST-CURRENT-STATE-212-20260722T145351Z.txt
1784730859.9011235460 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z-latest.txt
1784730859.8981025540 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.json
1784730859.8809036410 HOMELAB-TAILS-CLOSEOUT-211-20260722T143419Z.txt
1784730606.0033666560 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z-latest.txt
1784730606.0015773800 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.json
1784730605.9797819180 HOMELAB-TAILS-CLOSEOUT-210-20260722T143005Z.txt
1784730456.7690908780 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z-latest.txt
1784730456.7669407010 HOMELAB-TAILS-CLOSEOUT-209-20260722T142736Z.json
CHANGES_MADE=NO
OUTPUT_END
CHAT_OUTPUT_END