homelab-runtime-publisher
102 lines
8.2 KiB
Plaintext
102 lines
8.2 KiB
Plaintext
CHAT_OUTPUT_BEGIN
|
|
COMMAND_ID=HOMELAB-TERMINAL-CLOSURE-135
|
|
STATUS=FAIL
|
|
RC=42
|
|
HOST=pve01
|
|
MODE=read-only
|
|
COMPONENT=cluster-audit
|
|
REFERENCE_REGISTER_CHECK=OK
|
|
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
|
ERROR_REGISTER_CHECK=OK
|
|
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
|
COMMAND_SHA256=b3cd309d0bc46a8a9b521768bfe3180d01621663ea66cb68852eb378c0ba2714
|
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
|
EXECUTION_STARTED=true
|
|
CHANGES_MADE=false
|
|
SANITIZED=yes
|
|
SECRETS_INCLUDED=no
|
|
PRIVATE_ADDRESSES_INCLUDED=no
|
|
RAW_EVIDENCE_SHA256=93b6c0ea3e40a71162ebbec16e55264ca79609c36db3e6388dc5eb890851450a
|
|
SANITIZED_OUTPUT_SHA256=8d48421d14f766c054161a99ae23c23aeeed79e5f6d25bbc9cf273ef83562235
|
|
OUTPUT_BEGIN
|
|
JSON_IMMUTABLE_COMMIT=de867fb672a4d803e57934e8cf33a0da4901bfbb
|
|
TXT_IMMUTABLE_COMMIT=de867fb672a4d803e57934e8cf33a0da4901bfbb
|
|
=== IMMUTABLE_134 ===
|
|
COMMAND_ID=HOMELAB-FINAL-VERDICT-134
|
|
STATUS=OK
|
|
RC=0
|
|
EXECUTION_STARTED=true
|
|
ERROR_REGISTERED=no
|
|
INHERITED_VERDICT=BLOCKED
|
|
INHERITED_WORKFLOW=REQUIRES_ACTION
|
|
INHERITED_NEXT_ACTION=REPAIR_COMMAND_130_ONLY
|
|
=== INDEPENDENT_LIVE_GATE ===
|
|
PROBE_FAILURES=0
|
|
QUORUM_RISKS=0
|
|
FAILED_UNIT_RISKS=3
|
|
STORAGE_STATE_RISKS=0
|
|
CAPACITY_RISKS=0
|
|
KERNEL_RISKS=0
|
|
BACKUP_POLICY_RISKS=6
|
|
LOCAL_BACKUP_ARCHIVES=12
|
|
TOTAL_FINAL_RISKS=21
|
|
HOMELAB_TERMINAL_RESULT=BLOCKED
|
|
HOMELAB_WORKFLOW_STATUS=REQUIRES_ACTION
|
|
NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS
|
|
=== BLOCKER_EVIDENCE ===
|
|
Cluster information
|
|
-------------------
|
|
Name: homelab
|
|
Config Version: 3
|
|
Transport: knet
|
|
Secure auth: on
|
|
|
|
Quorum information
|
|
------------------
|
|
Date: Tue Jul 21 22:48:26 2026
|
|
Quorum provider: corosync_votequorum
|
|
Nodes: 3
|
|
Node ID: 0x00000001
|
|
Ring ID: 1.9d
|
|
Quorate: Yes
|
|
|
|
Votequorum information
|
|
----------------------
|
|
Expected votes: 3
|
|
Highest expected: 3
|
|
Total votes: 3
|
|
Quorum: 2
|
|
Flags: Quorate
|
|
|
|
Membership information
|
|
----------------------
|
|
Nodeid Votes Name
|
|
0x00000001 1 [PRIVATE_IP] (local)
|
|
0x00000002 1 [PRIVATE_IP]
|
|
0x00000003 1 [PRIVATE_IP]
|
|
homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state
|
|
homelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01
|
|
homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03
|
|
INFO: starting new backup job: vzdump 110 111 112 113 130 150 190 --mailnotification failure --prune-backups 'keep-daily=4' --mode snapshot --storage local --compress zstd --quiet 1
|
|
STATUS=WARN TS=2026-07-19T05:14:41Z TYPE=overall-health BAD=5 backup-framework.txt=OK drift-check.txt=WARN service-registry.txt=OK dependency-map.txt=OK golden-state.txt=OK cluster-passport.txt=WARN final-readiness.txt=WARN safe-autoheal.txt=OK kuma-monitor-policy.txt=OK backup-sla.txt=OK backup-coverage-matrix.txt=OK extended-appbackup.txt=OK residual-review.txt=OK forum-snuffleupagus.txt=OK incident-journal.txt=OK duty-admin-v2.txt=OK node-loss-readiness.txt=OK node-loss-runbooks.txt=OK power-loss-readiness.txt=OK power-loss-runbook.txt=OK ungated-health-backlog.txt=OK pve03-root-cleanup.txt=OK pve03-staging-retention.txt=OK pve03-staging-retention-proof.txt=OK pve03-staging-retention-cleanup.txt=OK pve03-failed-unit-cleanup.txt=OK remote-git-sops-age-readiness.txt=OK remote-git-sops-age-policy.txt=OK desired-state-remote-target-trace.txt=OK desired-state-remote-target.txt=OK desired-state.txt=OK runbooks.txt=OK alerting.txt=OK secret-exposure.txt=OK capacity-risk.txt=OK vm-local-dumps-retention.txt=OK vm-local-dumps-cloud.txt=OK vm-backup-policy.txt=OK vm170-vm171-full-strategy.txt=OK external-probe-vps.txt=OK netbird-vps-trust.txt=OK netbird-vps-trust-closure.txt=OK external-probe-runner-decision.txt=OK netbird-vps-trust-clean-seal.txt=OK live-tail-audit.txt=OK vm-backup.txt=OK cluster-admin-observer.txt=OK cluster-admin-restricted-probes.txt=WARN cluster-admin-full-observer.txt=WARN cluster-admin-vm-mail-cloud-backup.txt=OK cluster-admin-webpanel-sync.txt=OK cluster-admin-webpanel.txt=OK
|
|
INFO: starting new backup job: vzdump 110 111 112 113 130 150 190 --prune-backups 'keep-daily=4' --mode snapshot --mailnotification failure --storage local --compress zstd --quiet 1
|
|
STATUS=WARN TS=2026-07-20T05:10:25Z TYPE=overall-health BAD=5 backup-framework.txt=OK drift-check.txt=WARN service-registry.txt=OK dependency-map.txt=OK golden-state.txt=OK cluster-passport.txt=WARN final-readiness.txt=WARN safe-autoheal.txt=OK kuma-monitor-policy.txt=OK backup-sla.txt=OK backup-coverage-matrix.txt=OK extended-appbackup.txt=OK residual-review.txt=OK forum-snuffleupagus.txt=OK incident-journal.txt=OK duty-admin-v2.txt=OK node-loss-readiness.txt=OK node-loss-runbooks.txt=OK power-loss-readiness.txt=OK power-loss-runbook.txt=OK ungated-health-backlog.txt=OK pve03-root-cleanup.txt=OK pve03-staging-retention.txt=OK pve03-staging-retention-proof.txt=OK pve03-staging-retention-cleanup.txt=OK pve03-failed-unit-cleanup.txt=OK remote-git-sops-age-readiness.txt=OK remote-git-sops-age-policy.txt=OK desired-state-remote-target-trace.txt=OK desired-state-remote-target.txt=OK desired-state.txt=OK runbooks.txt=OK alerting.txt=OK secret-exposure.txt=OK capacity-risk.txt=OK vm-local-dumps-retention.txt=OK vm-local-dumps-cloud.txt=OK vm-backup-policy.txt=OK vm170-vm171-full-strategy.txt=OK external-probe-vps.txt=OK netbird-vps-trust.txt=OK netbird-vps-trust-closure.txt=OK external-probe-runner-decision.txt=OK netbird-vps-trust-clean-seal.txt=OK live-tail-audit.txt=OK vm-backup.txt=OK cluster-admin-observer.txt=OK cluster-admin-restricted-probes.txt=WARN cluster-admin-full-observer.txt=WARN cluster-admin-vm-mail-cloud-backup.txt=OK cluster-admin-webpanel-sync.txt=OK cluster-admin-webpanel.txt=OK
|
|
INFO: starting new backup job: vzdump 110 111 112 113 130 150 190 --compress zstd --quiet 1 --mode snapshot --prune-backups 'keep-daily=4' --mailnotification failure --storage local
|
|
STATUS=WARN TS=2026-07-21T05:01:14Z TYPE=overall-health BAD=7 backup-framework.txt=OK drift-check.txt=WARN service-registry.txt=OK dependency-map.txt=OK golden-state.txt=OK cluster-passport.txt=WARN final-readiness.txt=WARN safe-autoheal.txt=OK kuma-monitor-policy.txt=OK backup-sla.txt=WARN backup-coverage-matrix.txt=OK extended-appbackup.txt=OK residual-review.txt=OK forum-snuffleupagus.txt=OK incident-journal.txt=OK duty-admin-v2.txt=OK node-loss-readiness.txt=OK node-loss-runbooks.txt=OK power-loss-readiness.txt=OK power-loss-runbook.txt=OK ungated-health-backlog.txt=OK pve03-root-cleanup.txt=OK pve03-staging-retention.txt=OK pve03-staging-retention-proof.txt=OK pve03-staging-retention-cleanup.txt=OK pve03-failed-unit-cleanup.txt=OK remote-git-sops-age-readiness.txt=OK remote-git-sops-age-policy.txt=OK desired-state-remote-target-trace.txt=OK desired-state-remote-target.txt=OK desired-state.txt=OK runbooks.txt=OK alerting.txt=OK secret-exposure.txt=OK capacity-risk.txt=OK vm-local-dumps-retention.txt=OK vm-local-dumps-cloud.txt=OK vm-backup-policy.txt=OK vm170-vm171-full-strategy.txt=OK external-probe-vps.txt=OK netbird-vps-trust.txt=OK netbird-vps-trust-closure.txt=OK external-probe-runner-decision.txt=OK netbird-vps-trust-clean-seal.txt=OK live-tail-audit.txt=OK vm-backup.txt=OK cluster-admin-observer.txt=WARN cluster-admin-restricted-probes.txt=WARN cluster-admin-full-observer.txt=WARN cluster-admin-vm-mail-cloud-backup.txt=OK cluster-admin-webpanel-sync.txt=OK cluster-admin-webpanel.txt=OK
|
|
/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst
|
|
/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst
|
|
/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst
|
|
/var/lib/vz/dump/vzdump-lxc-110-2026_07_18-03_30_00.tar.zst
|
|
/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst
|
|
/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst
|
|
/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst
|
|
/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst
|
|
/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst
|
|
/var/lib/vz/dump/vzdump-qemu-150-2026_07_18-03_30_13.vma.zst
|
|
/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst
|
|
/var/lib/vz/dump/vzdump-lxc-112-2026_07_18-03_30_08.tar.zst
|
|
HOMELAB_TERMINAL_CLOSURE=BLOCKED
|
|
|
|
OUTPUT_END
|
|
CHAT_OUTPUT_END
|