homelab-runtime-publisher
166 lines
12 KiB
Plaintext
166 lines
12 KiB
Plaintext
CHAT_OUTPUT_BEGIN
|
|
COMMAND_ID=BACKUP-FAILED-UNITS-ROOTCAUSE-214
|
|
STATUS=OK
|
|
RC=0
|
|
HOST=pve01
|
|
MODE=read-only
|
|
COMPONENT=backup-audit
|
|
REFERENCE_REGISTER_CHECK=OK
|
|
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
|
ERROR_REGISTER_CHECK=OK
|
|
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
|
COMMAND_SHA256=19d600f59a44903ae388de5924789d15c9dda0ede9ed5f3966600f8f808c43c1
|
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
|
EXECUTION_STARTED=true
|
|
CHANGES_MADE=false
|
|
SANITIZED=yes
|
|
SECRETS_INCLUDED=no
|
|
PRIVATE_ADDRESSES_INCLUDED=no
|
|
RAW_EVIDENCE_SHA256=342cecee34231056a87789ec2411b4a4e54071817393a6b227a5a74f2e6619c3
|
|
SANITIZED_OUTPUT_SHA256=d51060b6b35d2cbb1667bc9bee8d6bc8dc2079c25fb2dcf949eda14fa37537cc
|
|
OUTPUT_BEGIN
|
|
PREVIOUS_COMMAND_ID=BACKUP-OFFHOST-DELTA-212
|
|
MISSING_RESULT_213=CONFIRMED
|
|
===== homelab-backup-matrix-refresh =====
|
|
Result=exit-code
|
|
ExecMainStatus=1
|
|
ActiveState=failed
|
|
SubState=failed
|
|
MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
|
|
homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
|
|
Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
|
|
MATRIX_STATE_STATUS=FAIL
|
|
MATRIX_STATE_JOBS_TOTAL=41
|
|
MATRIX_STATE_JOBS_OK=30
|
|
MATRIX_STATE_JOBS_BAD=6
|
|
MATRIX_STATE_JOBS_UNMAPPED=0
|
|
MATRIX_STATE_JOBS_DISABLED=5
|
|
MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
|
|
MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
|
|
homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
|
|
Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
|
|
MATRIX_STATE_STATUS=FAIL
|
|
MATRIX_STATE_JOBS_TOTAL=41
|
|
MATRIX_STATE_JOBS_OK=30
|
|
MATRIX_STATE_JOBS_BAD=6
|
|
MATRIX_STATE_JOBS_UNMAPPED=0
|
|
MATRIX_STATE_JOBS_DISABLED=5
|
|
MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
|
|
MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
|
|
homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
|
|
Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
|
|
MATRIX_STATE_STATUS=FAIL
|
|
MATRIX_STATE_JOBS_TOTAL=41
|
|
MATRIX_STATE_JOBS_OK=30
|
|
MATRIX_STATE_JOBS_BAD=6
|
|
MATRIX_STATE_JOBS_UNMAPPED=0
|
|
MATRIX_STATE_JOBS_DISABLED=5
|
|
MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
|
|
MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
|
|
homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
|
|
===== homelab-vm160-remote-backup =====
|
|
Result=exit-code
|
|
ExecMainStatus=1
|
|
ActiveState=failed
|
|
SubState=failed
|
|
homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak.
|
|
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
|
|
VM160_TARGET_FINALIZE_STATUS=OK
|
|
VM160_TARGET_PROMOTE_STATUS=OK
|
|
VM160_REMOTE_BACKUP_STATUS=OK
|
|
TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z
|
|
ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst
|
|
SIZE_BYTES=84995216465
|
|
SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72
|
|
homelab-vm160-remote-backup.service: Deactivated successfully.
|
|
Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
|
|
homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak.
|
|
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
|
|
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
|
|
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
|
|
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
|
|
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
|
|
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
|
|
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
|
|
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
|
|
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
|
|
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
|
|
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
|
|
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
|
|
Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
|
|
homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
|
|
===== homelab-cluster-admin-pull =====
|
|
Result=exit-code
|
|
ExecMainStatus=1
|
|
ActiveState=failed
|
|
SubState=failed
|
|
STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
|
|
STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
|
|
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
|
|
Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
|
|
STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
|
|
STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
|
|
STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
|
|
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
|
|
Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
|
|
STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
|
|
STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
|
|
STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
|
|
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
|
|
Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
|
|
STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
|
|
STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
|
|
STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
|
|
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
|
|
Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
|
|
STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
|
|
STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
|
|
STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
|
|
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
|
|
Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
|
|
STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
|
|
STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
|
|
STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
|
|
homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
|
|
homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
|
|
Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
|
|
===== MATRIX_SYNTAX =====
|
|
/usr/local/sbin/homelab-backup-matrix-refresh: line 5: syntax error near unexpected token `('
|
|
/usr/local/sbin/homelab-backup-matrix-refresh: line 5: `M=Path("/etc/homelab-backup-matrix/matrix.json")'
|
|
MATRIX_SYNTAX_RC=2
|
|
|
|
OUTPUT_END
|
|
CHAT_OUTPUT_END
|