runtime: publish BACKUP-FAILED-UNITS-EVIDENCE-219

runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-219.json

@@ -0,0 +1,30 @@
+{
+  "schema_version": 1,
+  "channel": "homelab-runtime",
+  "command_id": "BACKUP-FAILED-UNITS-EVIDENCE-219",
+  "status": "OK",
+  "rc": 0,
+  "host": "pve01",
+  "mode": "read-only",
+  "component": "cluster-audit",
+  "started_at_utc": "2026-07-22T15:42:52Z",
+  "finished_at_utc": "2026-07-22T15:42:52Z",
+  "reference_register_checked": true,
+  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
+  "error_register_checked": true,
+  "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
+  "command_sha256": "250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2",
+  "duplicate_failed_command_blocked": false,
+  "block_reason": null,
+  "execution_started": true,
+  "changes_made": false,
+  "sanitized": true,
+  "secrets_included": false,
+  "private_addresses_included": false,
+  "raw_evidence_retained_locally": true,
+  "raw_evidence_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607",
+  "sanitized_output_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607",
+  "output_truncated_in_json": false,
+  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
+  "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218\n===LEDGER_218===\n2026-07-22T15:39:09Z\tBACKUP-FAILED-UNITS-DECISION-218\t67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730\t0\tOK\tcluster-audit\n===MATRIX_NON_OK===\n{\"id\":\"filebrowser_restore_validate\",\"name\":\"filebrowser-restore-validate\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"filebrowser-restore-validate.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\"}\n{\"id\":\"homelab_backup_sla_health\",\"name\":\"homelab-backup-sla-health\",\"status\":\"FAIL\",\"source_status\":\"WARN\",\"freshness\":\"FRESH\",\"service\":\"homelab-backup-sla-health.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\"}\n{\"id\":\"homelab_mail_cloud_restore_drill\",\"name\":\"homelab-mail-cloud-restore-drill\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"homelab-mail-cloud-restore-drill.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\"}\n{\"id\":\"netbird_vps_backup\",\"name\":\"netbird-vps-backup\",\"status\":\"FAIL\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"netbird-vps-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\"}\n{\"id\":\"paperless_restore_dry_run\",\"name\":\"paperless-restore-dry-run\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"paperless-restore-dry-run.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\"}\n{\"id\":\"vm160_remote_backup\",\"name\":\"vm160-remote-backup\",\"status\":\"FAIL\",\"source_status\":\"FAIL\",\"freshness\":\"FRESH\",\"service\":\"homelab-vm160-remote-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\"}\n===VM160_HEALTH===\nSTATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=20260720T233913Z\nVMID=160\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z\nRC=1\n===CLUSTER_ADMIN_FAILURES===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n"
+}

runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-219.txt

@@ -0,0 +1,45 @@
+CHAT_OUTPUT_BEGIN
+COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219
+STATUS=OK
+RC=0
+HOST=pve01
+MODE=read-only
+COMPONENT=cluster-audit
+REFERENCE_REGISTER_CHECK=OK
+REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
+ERROR_REGISTER_CHECK=OK
+ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
+COMMAND_SHA256=250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2
+DUPLICATE_FAILED_COMMAND_BLOCKED=false
+EXECUTION_STARTED=true
+CHANGES_MADE=false
+SANITIZED=yes
+SECRETS_INCLUDED=no
+PRIVATE_ADDRESSES_INCLUDED=no
+RAW_EVIDENCE_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607
+SANITIZED_OUTPUT_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607
+OUTPUT_BEGIN
+PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218
+===LEDGER_218===
+2026-07-22T15:39:09Z	BACKUP-FAILED-UNITS-DECISION-218	67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730	0	OK	cluster-audit
+===MATRIX_NON_OK===
+{"id":"filebrowser_restore_validate","name":"filebrowser-restore-validate","status":"STALE","source_status":"OK","freshness":"STALE","service":"filebrowser-restore-validate.service","source_health":"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt"}
+{"id":"homelab_backup_sla_health","name":"homelab-backup-sla-health","status":"FAIL","source_status":"WARN","freshness":"FRESH","service":"homelab-backup-sla-health.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt"}
+{"id":"homelab_mail_cloud_restore_drill","name":"homelab-mail-cloud-restore-drill","status":"STALE","source_status":"OK","freshness":"STALE","service":"homelab-mail-cloud-restore-drill.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt"}
+{"id":"netbird_vps_backup","name":"netbird-vps-backup","status":"FAIL","source_status":"OK","freshness":"STALE","service":"netbird-vps-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt"}
+{"id":"paperless_restore_dry_run","name":"paperless-restore-dry-run","status":"STALE","source_status":"OK","freshness":"STALE","service":"paperless-restore-dry-run.service","source_health":"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt"}
+{"id":"vm160_remote_backup","name":"vm160-remote-backup","status":"FAIL","source_status":"FAIL","freshness":"FRESH","service":"homelab-vm160-remote-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt"}
+===VM160_HEALTH===
+STATUS=FAIL
+SERVICE=homelab-vm160-remote-backup
+TIMESTAMP_UTC=20260720T233913Z
+VMID=160
+TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z
+RC=1
+===CLUSTER_ADMIN_FAILURES===
+/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
+/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
+CHANGES_MADE=NO
+
+OUTPUT_END
+CHAT_OUTPUT_END

runtime/latest.json

@@ -1,19 +1,19 @@
 {
   "schema_version": 1,
   "channel": "homelab-runtime",
-  "command_id": "BACKUP-FAILED-UNITS-DECISION-218",
+  "command_id": "BACKUP-FAILED-UNITS-EVIDENCE-219",
   "status": "OK",
   "rc": 0,
   "host": "pve01",
   "mode": "read-only",
   "component": "cluster-audit",
-  "started_at_utc": "2026-07-22T15:39:09Z",
+  "started_at_utc": "2026-07-22T15:42:52Z",
-  "finished_at_utc": "2026-07-22T15:39:09Z",
+  "finished_at_utc": "2026-07-22T15:42:52Z",
   "reference_register_checked": true,
   "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
   "error_register_checked": true,
   "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
-  "command_sha256": "67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730",
+  "command_sha256": "250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2",
   "duplicate_failed_command_blocked": false,
   "block_reason": null,
   "execution_started": true,
@@ -22,9 +22,9 @@
   "secrets_included": false,
   "private_addresses_included": false,
   "raw_evidence_retained_locally": true,
-  "raw_evidence_sha256": "ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1",
+  "raw_evidence_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607",
-  "sanitized_output_sha256": "d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733",
+  "sanitized_output_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607",
   "output_truncated_in_json": false,
   "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
-  "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217\n=== LEDGER_217 ===\n2026-07-22T15:35:58Z\tBACKUP-FAILED-UNITS-SOURCE-217\t2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86\t0\tOK\tcluster-audit\n=== MATRIX_ENTRYPOINT ===\n     1\t#!/usr/bin/env python3\n     2\tfrom pathlib import Path\n     3\timport shutil\n     4\timport json,subprocess,os,time,sys,hashlib\n     5\tM=Path(\"/etc/homelab-backup-matrix/matrix.json\")\n     6\tROOT=Path(\"/var/lib/homelab-backup-matrix\")\n     7\tRENDER=ROOT/\"rendered\"\n     8\tCOMPAT=ROOT/\"compat\"\n     9\tdef cmd(args):\n    10\t    p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)\n    11\t    return p.stdout.strip() or \"unknown\"\n    12\tdef atomic(path,text,mode=0o640):\n    13\t    path.parent.mkdir(parents=True,exist_ok=True)\n    14\t    tmp=path.with_name(path.name+\".tmp\")\n    15\t    tmp.write_text(text,encoding=\"utf-8\")\n    16\t    os.chmod(tmp,mode)\n    17\t    os.replace(tmp,path)\n    18\tdef parse_health(path):\n=== VM160_CONTROL_FLOW ===\n5:SOURCE=root@[PRIVATE_IP]\n6:TARGET=root@[PRIVATE_IP]\n7:KH=[SENSITIVE_PATH]\n8:PARENT=/mnt/staging/vzdump\n11:LOCK=/run/lock/homelab-vm160-remote-backup.lock\n13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker\n14:target_worker(){ ssh \"${SSH[@]}\" \"$TARGET\" bash -s -- \"$@\" < \"$TARGET_WORKER\"; }\n15:install -d -m 0755 \"$(dirname \"$HEALTH\")\" \"$(dirname \"$LOCK\")\"\n16:exec 9>\"$LOCK\"\n19:test -x \"$TARGET_WORKER\"\n20:test \"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"qm status $VMID\" | awk '{print $2}')\" = running\n21:SOURCE_ACTIVE_RC=0\n22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n23:test \"$SOURCE_ACTIVE_RC\" -eq 0 || test \"$SOURCE_ACTIVE_RC\" -eq 1\n24:test -z \"$SOURCE_ACTIVE\"\n27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n35:    ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n37:    printf 'STATUS=FAIL\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nTARGET_DIR=%s\\nRC=%s\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$rc\" > \"$tmp\"\n44:trap cleanup EXIT\n45:REMOTE_FREE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"df -B1 --output=avail '$PARENT' | awk 'NR==2{print \\$1}'\")\"\n46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n48:if test \"$LATEST_SIZE\" -gt 0; then\n50:else\n52:fi\n53:test \"$REMOTE_FREE\" -ge \"$REQUIRED\"\n54:ssh \"${SSH[@]}\" \"$TARGET\" \"install -d -m 0700 '$DIR'\"\n55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n58:SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"stat -c %s '$FINAL'\")\"\n59:SHA=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"sha256sum '$FINAL' | awk '{print \\$1}'\")\"\n62:printf 'STATUS=OK\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nSOURCE_NODE=pve02\\nTARGET_NODE=pve03\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\nREMOTE_RETENTION_GENERATIONS=1\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\" > \"$tmp\"\n65:trap - EXIT\n67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\n' \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\"\n=== CLUSTER_ADMIN_CURRENT_FAILURE ===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n"
+  "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218\n===LEDGER_218===\n2026-07-22T15:39:09Z\tBACKUP-FAILED-UNITS-DECISION-218\t67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730\t0\tOK\tcluster-audit\n===MATRIX_NON_OK===\n{\"id\":\"filebrowser_restore_validate\",\"name\":\"filebrowser-restore-validate\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"filebrowser-restore-validate.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\"}\n{\"id\":\"homelab_backup_sla_health\",\"name\":\"homelab-backup-sla-health\",\"status\":\"FAIL\",\"source_status\":\"WARN\",\"freshness\":\"FRESH\",\"service\":\"homelab-backup-sla-health.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\"}\n{\"id\":\"homelab_mail_cloud_restore_drill\",\"name\":\"homelab-mail-cloud-restore-drill\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"homelab-mail-cloud-restore-drill.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\"}\n{\"id\":\"netbird_vps_backup\",\"name\":\"netbird-vps-backup\",\"status\":\"FAIL\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"netbird-vps-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\"}\n{\"id\":\"paperless_restore_dry_run\",\"name\":\"paperless-restore-dry-run\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"paperless-restore-dry-run.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\"}\n{\"id\":\"vm160_remote_backup\",\"name\":\"vm160-remote-backup\",\"status\":\"FAIL\",\"source_status\":\"FAIL\",\"freshness\":\"FRESH\",\"service\":\"homelab-vm160-remote-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\"}\n===VM160_HEALTH===\nSTATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=20260720T233913Z\nVMID=160\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z\nRC=1\n===CLUSTER_ADMIN_FAILURES===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n"
 }

runtime/latest.txt

@@ -1,5 +1,5 @@
 CHAT_OUTPUT_BEGIN
-COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218
+COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219
 STATUS=OK
 RC=0
 HOST=pve01
@@ -9,73 +9,34 @@ REFERENCE_REGISTER_CHECK=OK
 REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
 ERROR_REGISTER_CHECK=OK
 ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
-COMMAND_SHA256=67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730
+COMMAND_SHA256=250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2
 DUPLICATE_FAILED_COMMAND_BLOCKED=false
 EXECUTION_STARTED=true
 CHANGES_MADE=false
 SANITIZED=yes
 SECRETS_INCLUDED=no
 PRIVATE_ADDRESSES_INCLUDED=no
-RAW_EVIDENCE_SHA256=ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1
+RAW_EVIDENCE_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607
-SANITIZED_OUTPUT_SHA256=d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733
+SANITIZED_OUTPUT_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607
 OUTPUT_BEGIN
-PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217
+PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218
-=== LEDGER_217 ===
+===LEDGER_218===
-2026-07-22T15:35:58Z	BACKUP-FAILED-UNITS-SOURCE-217	2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86	0	OK	cluster-audit
+2026-07-22T15:39:09Z	BACKUP-FAILED-UNITS-DECISION-218	67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730	0	OK	cluster-audit
-=== MATRIX_ENTRYPOINT ===
+===MATRIX_NON_OK===
-     1	#!/usr/bin/env python3
+{"id":"filebrowser_restore_validate","name":"filebrowser-restore-validate","status":"STALE","source_status":"OK","freshness":"STALE","service":"filebrowser-restore-validate.service","source_health":"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt"}
-     2	from pathlib import Path
+{"id":"homelab_backup_sla_health","name":"homelab-backup-sla-health","status":"FAIL","source_status":"WARN","freshness":"FRESH","service":"homelab-backup-sla-health.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt"}
-     3	import shutil
+{"id":"homelab_mail_cloud_restore_drill","name":"homelab-mail-cloud-restore-drill","status":"STALE","source_status":"OK","freshness":"STALE","service":"homelab-mail-cloud-restore-drill.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt"}
-     4	import json,subprocess,os,time,sys,hashlib
+{"id":"netbird_vps_backup","name":"netbird-vps-backup","status":"FAIL","source_status":"OK","freshness":"STALE","service":"netbird-vps-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt"}
-     5	M=Path("/etc/homelab-backup-matrix/matrix.json")
+{"id":"paperless_restore_dry_run","name":"paperless-restore-dry-run","status":"STALE","source_status":"OK","freshness":"STALE","service":"paperless-restore-dry-run.service","source_health":"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt"}
-     6	ROOT=Path("/var/lib/homelab-backup-matrix")
+{"id":"vm160_remote_backup","name":"vm160-remote-backup","status":"FAIL","source_status":"FAIL","freshness":"FRESH","service":"homelab-vm160-remote-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt"}
-     7	RENDER=ROOT/"rendered"
+===VM160_HEALTH===
-     8	COMPAT=ROOT/"compat"
+STATUS=FAIL
-     9	def cmd(args):
+SERVICE=homelab-vm160-remote-backup
-    10	    p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)
+TIMESTAMP_UTC=20260720T233913Z
-    11	    return p.stdout.strip() or "unknown"
+VMID=160
-    12	def atomic(path,text,mode=0o640):
+TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z
-    13	    path.parent.mkdir(parents=True,exist_ok=True)
+RC=1
-    14	    tmp=path.with_name(path.name+".tmp")
+===CLUSTER_ADMIN_FAILURES===
-    15	    tmp.write_text(text,encoding="utf-8")
-    16	    os.chmod(tmp,mode)
-    17	    os.replace(tmp,path)
-    18	def parse_health(path):
-=== VM160_CONTROL_FLOW ===
-5:SOURCE=root@[PRIVATE_IP]
-6:TARGET=root@[PRIVATE_IP]
-7:KH=[SENSITIVE_PATH]
-8:PARENT=/mnt/staging/vzdump
-11:LOCK=/run/lock/homelab-vm160-remote-backup.lock
-13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker
-14:target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; }
-15:install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")"
-16:exec 9>"$LOCK"
-19:test -x "$TARGET_WORKER"
-20:test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running
-21:SOURCE_ACTIVE_RC=0
-22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$?
-23:test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1
-24:test -z "$SOURCE_ACTIVE"
-27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst"
-35:    ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true
-37:    printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp"
-44:trap cleanup EXIT
-45:REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")"
-46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")"
-48:if test "$LATEST_SIZE" -gt 0; then
-50:else
-52:fi
-53:test "$REMOTE_FREE" -ge "$REQUIRED"
-54:ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'"
-55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'"
-57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG"
-58:SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")"
-59:SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")"
-62:printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp"
-65:trap - EXIT
-67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA"
-=== CLUSTER_ADMIN_CURRENT_FAILURE ===
 /var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
 /var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
 CHANGES_MADE=NO