runtime: publish BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220

runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.json

@@ -0,0 +1,30 @@
+{
+  "schema_version": 1,
+  "channel": "homelab-runtime",
+  "command_id": "BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220",
+  "status": "OK",
+  "rc": 0,
+  "host": "pve01",
+  "mode": "read-only",
+  "component": "cluster-audit",
+  "started_at_utc": "2026-07-22T15:46:24Z",
+  "finished_at_utc": "2026-07-22T15:46:24Z",
+  "reference_register_checked": true,
+  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
+  "error_register_checked": true,
+  "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
+  "command_sha256": "728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea",
+  "duplicate_failed_command_blocked": false,
+  "block_reason": null,
+  "execution_started": true,
+  "changes_made": false,
+  "sanitized": true,
+  "secrets_included": false,
+  "private_addresses_included": false,
+  "raw_evidence_retained_locally": true,
+  "raw_evidence_sha256": "7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331",
+  "sanitized_output_sha256": "7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331",
+  "output_truncated_in_json": false,
+  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
+  "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219\n=== LEDGER_219 ===\n2026-07-22T15:42:52Z\tBACKUP-FAILED-UNITS-EVIDENCE-219\t250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2\t0\tOK\tcluster-audit\n=== IMMUTABLE_LOCAL_RESULT_219 ===\nLOCAL_RESULT_219=MISSING\nCHANGES_MADE=NO\n"
+}

runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.txt

@@ -0,0 +1,30 @@
+CHAT_OUTPUT_BEGIN
+COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220
+STATUS=OK
+RC=0
+HOST=pve01
+MODE=read-only
+COMPONENT=cluster-audit
+REFERENCE_REGISTER_CHECK=OK
+REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
+ERROR_REGISTER_CHECK=OK
+ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
+COMMAND_SHA256=728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea
+DUPLICATE_FAILED_COMMAND_BLOCKED=false
+EXECUTION_STARTED=true
+CHANGES_MADE=false
+SANITIZED=yes
+SECRETS_INCLUDED=no
+PRIVATE_ADDRESSES_INCLUDED=no
+RAW_EVIDENCE_SHA256=7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331
+SANITIZED_OUTPUT_SHA256=7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331
+OUTPUT_BEGIN
+PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219
+=== LEDGER_219 ===
+2026-07-22T15:42:52Z	BACKUP-FAILED-UNITS-EVIDENCE-219	250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2	0	OK	cluster-audit
+=== IMMUTABLE_LOCAL_RESULT_219 ===
+LOCAL_RESULT_219=MISSING
+CHANGES_MADE=NO
+
+OUTPUT_END
+CHAT_OUTPUT_END

runtime/latest.json

@@ -1,19 +1,19 @@
 {
   "schema_version": 1,
   "channel": "homelab-runtime",
-  "command_id": "BACKUP-FAILED-UNITS-EVIDENCE-219",
+  "command_id": "BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220",
   "status": "OK",
   "rc": 0,
   "host": "pve01",
   "mode": "read-only",
   "component": "cluster-audit",
-  "started_at_utc": "2026-07-22T15:42:52Z",
+  "started_at_utc": "2026-07-22T15:46:24Z",
-  "finished_at_utc": "2026-07-22T15:42:52Z",
+  "finished_at_utc": "2026-07-22T15:46:24Z",
   "reference_register_checked": true,
   "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
   "error_register_checked": true,
   "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
-  "command_sha256": "250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2",
+  "command_sha256": "728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea",
   "duplicate_failed_command_blocked": false,
   "block_reason": null,
   "execution_started": true,
@@ -22,9 +22,9 @@
   "secrets_included": false,
   "private_addresses_included": false,
   "raw_evidence_retained_locally": true,
-  "raw_evidence_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607",
+  "raw_evidence_sha256": "7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331",
-  "sanitized_output_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607",
+  "sanitized_output_sha256": "7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331",
   "output_truncated_in_json": false,
   "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
-  "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218\n===LEDGER_218===\n2026-07-22T15:39:09Z\tBACKUP-FAILED-UNITS-DECISION-218\t67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730\t0\tOK\tcluster-audit\n===MATRIX_NON_OK===\n{\"id\":\"filebrowser_restore_validate\",\"name\":\"filebrowser-restore-validate\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"filebrowser-restore-validate.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\"}\n{\"id\":\"homelab_backup_sla_health\",\"name\":\"homelab-backup-sla-health\",\"status\":\"FAIL\",\"source_status\":\"WARN\",\"freshness\":\"FRESH\",\"service\":\"homelab-backup-sla-health.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\"}\n{\"id\":\"homelab_mail_cloud_restore_drill\",\"name\":\"homelab-mail-cloud-restore-drill\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"homelab-mail-cloud-restore-drill.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\"}\n{\"id\":\"netbird_vps_backup\",\"name\":\"netbird-vps-backup\",\"status\":\"FAIL\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"netbird-vps-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\"}\n{\"id\":\"paperless_restore_dry_run\",\"name\":\"paperless-restore-dry-run\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"paperless-restore-dry-run.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\"}\n{\"id\":\"vm160_remote_backup\",\"name\":\"vm160-remote-backup\",\"status\":\"FAIL\",\"source_status\":\"FAIL\",\"freshness\":\"FRESH\",\"service\":\"homelab-vm160-remote-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\"}\n===VM160_HEALTH===\nSTATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=20260720T233913Z\nVMID=160\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z\nRC=1\n===CLUSTER_ADMIN_FAILURES===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n"
+  "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219\n=== LEDGER_219 ===\n2026-07-22T15:42:52Z\tBACKUP-FAILED-UNITS-EVIDENCE-219\t250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2\t0\tOK\tcluster-audit\n=== IMMUTABLE_LOCAL_RESULT_219 ===\nLOCAL_RESULT_219=MISSING\nCHANGES_MADE=NO\n"
 }

runtime/latest.txt

@@ -1,5 +1,5 @@
 CHAT_OUTPUT_BEGIN
-COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219
+COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220
 STATUS=OK
 RC=0
 HOST=pve01
@@ -9,36 +9,21 @@ REFERENCE_REGISTER_CHECK=OK
 REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
 ERROR_REGISTER_CHECK=OK
 ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
-COMMAND_SHA256=250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2
+COMMAND_SHA256=728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea
 DUPLICATE_FAILED_COMMAND_BLOCKED=false
 EXECUTION_STARTED=true
 CHANGES_MADE=false
 SANITIZED=yes
 SECRETS_INCLUDED=no
 PRIVATE_ADDRESSES_INCLUDED=no
-RAW_EVIDENCE_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607
+RAW_EVIDENCE_SHA256=7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331
-SANITIZED_OUTPUT_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607
+SANITIZED_OUTPUT_SHA256=7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331
 OUTPUT_BEGIN
-PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218
+PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219
-===LEDGER_218===
+=== LEDGER_219 ===
-2026-07-22T15:39:09Z	BACKUP-FAILED-UNITS-DECISION-218	67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730	0	OK	cluster-audit
+2026-07-22T15:42:52Z	BACKUP-FAILED-UNITS-EVIDENCE-219	250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2	0	OK	cluster-audit
-===MATRIX_NON_OK===
+=== IMMUTABLE_LOCAL_RESULT_219 ===
-{"id":"filebrowser_restore_validate","name":"filebrowser-restore-validate","status":"STALE","source_status":"OK","freshness":"STALE","service":"filebrowser-restore-validate.service","source_health":"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt"}
+LOCAL_RESULT_219=MISSING
-{"id":"homelab_backup_sla_health","name":"homelab-backup-sla-health","status":"FAIL","source_status":"WARN","freshness":"FRESH","service":"homelab-backup-sla-health.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt"}
-{"id":"homelab_mail_cloud_restore_drill","name":"homelab-mail-cloud-restore-drill","status":"STALE","source_status":"OK","freshness":"STALE","service":"homelab-mail-cloud-restore-drill.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt"}
-{"id":"netbird_vps_backup","name":"netbird-vps-backup","status":"FAIL","source_status":"OK","freshness":"STALE","service":"netbird-vps-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt"}
-{"id":"paperless_restore_dry_run","name":"paperless-restore-dry-run","status":"STALE","source_status":"OK","freshness":"STALE","service":"paperless-restore-dry-run.service","source_health":"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt"}
-{"id":"vm160_remote_backup","name":"vm160-remote-backup","status":"FAIL","source_status":"FAIL","freshness":"FRESH","service":"homelab-vm160-remote-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt"}
-===VM160_HEALTH===
-STATUS=FAIL
-SERVICE=homelab-vm160-remote-backup
-TIMESTAMP_UTC=20260720T233913Z
-VMID=160
-TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z
-RC=1
-===CLUSTER_ADMIN_FAILURES===
-/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
-/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
 CHANGES_MADE=NO
 
 OUTPUT_END