runtime: publish BACKUP-FAILED-UNITS-RESULT-222

runtime/history/BACKUP-FAILED-UNITS-RESULT-222.json

@@ -0,0 +1,30 @@
+{
+  "schema_version": 1,
+  "channel": "homelab-runtime",
+  "command_id": "BACKUP-FAILED-UNITS-RESULT-222",
+  "status": "OK",
+  "rc": 0,
+  "host": "pve01",
+  "mode": "read-only",
+  "component": "cluster-audit",
+  "started_at_utc": "2026-07-22T15:56:39Z",
+  "finished_at_utc": "2026-07-22T15:56:39Z",
+  "reference_register_checked": true,
+  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
+  "error_register_checked": true,
+  "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
+  "command_sha256": "c1f54adfcc646548cd97343aa00aa80749ea640f6d3a077cebc664e582d3e591",
+  "duplicate_failed_command_blocked": false,
+  "block_reason": null,
+  "execution_started": true,
+  "changes_made": false,
+  "sanitized": true,
+  "secrets_included": false,
+  "private_addresses_included": false,
+  "raw_evidence_retained_locally": true,
+  "raw_evidence_sha256": "f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23",
+  "sanitized_output_sha256": "f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23",
+  "output_truncated_in_json": false,
+  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
+  "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-PREFLIGHT-221\n=== LEDGER_221 ===\n2026-07-22T15:50:09Z\tBACKUP-FAILED-UNITS-PREFLIGHT-221\ta2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8\t0\tOK\tcluster-audit\n=== EXACT_IMMUTABLE_RESULT_221 ===\nRESULT_221=MISSING_LOCAL\n=== CURRENT_FAILED_UNITS ===\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nhomelab-cluster-admin-pull.service    loaded failed failed Pull cluster-admin observer health into pve01\nhomelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03\nCHANGES_MADE=NO\n"
+}

runtime/history/BACKUP-FAILED-UNITS-RESULT-222.txt

@@ -0,0 +1,34 @@
+CHAT_OUTPUT_BEGIN
+COMMAND_ID=BACKUP-FAILED-UNITS-RESULT-222
+STATUS=OK
+RC=0
+HOST=pve01
+MODE=read-only
+COMPONENT=cluster-audit
+REFERENCE_REGISTER_CHECK=OK
+REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
+ERROR_REGISTER_CHECK=OK
+ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
+COMMAND_SHA256=c1f54adfcc646548cd97343aa00aa80749ea640f6d3a077cebc664e582d3e591
+DUPLICATE_FAILED_COMMAND_BLOCKED=false
+EXECUTION_STARTED=true
+CHANGES_MADE=false
+SANITIZED=yes
+SECRETS_INCLUDED=no
+PRIVATE_ADDRESSES_INCLUDED=no
+RAW_EVIDENCE_SHA256=f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23
+SANITIZED_OUTPUT_SHA256=f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23
+OUTPUT_BEGIN
+PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-PREFLIGHT-221
+=== LEDGER_221 ===
+2026-07-22T15:50:09Z	BACKUP-FAILED-UNITS-PREFLIGHT-221	a2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8	0	OK	cluster-audit
+=== EXACT_IMMUTABLE_RESULT_221 ===
+RESULT_221=MISSING_LOCAL
+=== CURRENT_FAILED_UNITS ===
+homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state
+homelab-cluster-admin-pull.service    loaded failed failed Pull cluster-admin observer health into pve01
+homelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03
+CHANGES_MADE=NO
+
+OUTPUT_END
+CHAT_OUTPUT_END

runtime/latest.json

@@ -1,19 +1,19 @@
 {
   "schema_version": 1,
   "channel": "homelab-runtime",
-  "command_id": "BACKUP-FAILED-UNITS-PREFLIGHT-221",
+  "command_id": "BACKUP-FAILED-UNITS-RESULT-222",
   "status": "OK",
   "rc": 0,
   "host": "pve01",
   "mode": "read-only",
   "component": "cluster-audit",
-  "started_at_utc": "2026-07-22T15:50:08Z",
+  "started_at_utc": "2026-07-22T15:56:39Z",
-  "finished_at_utc": "2026-07-22T15:50:09Z",
+  "finished_at_utc": "2026-07-22T15:56:39Z",
   "reference_register_checked": true,
   "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
   "error_register_checked": true,
   "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
-  "command_sha256": "a2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8",
+  "command_sha256": "c1f54adfcc646548cd97343aa00aa80749ea640f6d3a077cebc664e582d3e591",
   "duplicate_failed_command_blocked": false,
   "block_reason": null,
   "execution_started": true,
@@ -22,9 +22,9 @@
   "secrets_included": false,
   "private_addresses_included": false,
   "raw_evidence_retained_locally": true,
-  "raw_evidence_sha256": "45b89b2bddc0a8064033309702ebcffe8b64609b2786bd3a6f4e0532491ac5e1",
+  "raw_evidence_sha256": "f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23",
-  "sanitized_output_sha256": "45b89b2bddc0a8064033309702ebcffe8b64609b2786bd3a6f4e0532491ac5e1",
+  "sanitized_output_sha256": "f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23",
   "output_truncated_in_json": false,
   "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
-  "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220\n=== LEDGER_219_220 ===\n2026-07-22T15:42:52Z\tBACKUP-FAILED-UNITS-EVIDENCE-219\t250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2\t0\tOK\tcluster-audit\n2026-07-22T15:46:24Z\tBACKUP-FAILED-UNITS-EVIDENCE-RELAY-220\t728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea\t0\tOK\tcluster-audit\n=== MATRIX_NON_OK ===\nfilebrowser_restore_validate\tfilebrowser-restore-validate\tSTALE\tenabled\tactive\tsuccess\t0\tOK\t1103338\t864000\tSTALE\t/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\nhomelab_backup_sla_health\thomelab-backup-sla-health\tFAIL\tenabled\tactive\tsuccess\t0\tWARN\t658\t172800\tFRESH\t/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\nhomelab_mail_cloud_restore_drill\thomelab-mail-cloud-restore-drill\tSTALE\tenabled\tactive\tsuccess\t0\tOK\t1075183\t864000\tSTALE\t/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\nnetbird_vps_backup\tnetbird-vps-backup\tFAIL\tdisabled\tinactive\tsuccess\t0\tOK\t1834810\t864000\tSTALE\t/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\npaperless_restore_dry_run\tpaperless-restore-dry-run\tSTALE\tenabled\tactive\tsuccess\t0\tOK\t1088900\t864000\tSTALE\t/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\nvm160_remote_backup\tvm160-remote-backup\tFAIL\tdisabled\tinactive\texit-code\t1\tFAIL\t113521\t172800\tFRESH\t/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\n=== VM160_PREFLIGHT ===\nHOST_CHECK_RC=0\nTARGET_WORKER_RC=0\nSOURCE_VM_RC=0\nstatus: running\nSOURCE_ACTIVE_RC=1\nSOURCE_IDLE_RC=0\nTARGET_FREE_QUERY_RC=0\nTARGET_FREE_BYTES=92632272896\nLATEST_SIZE_QUERY_RC=0\nLATEST_SIZE_BYTES=84995216465\nREQUIRED_BYTES=116981438821\nCAPACITY_CHECK_RC=1\n=== CLUSTER_ADMIN_PROBE_FAILURES ===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n"
+  "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-PREFLIGHT-221\n=== LEDGER_221 ===\n2026-07-22T15:50:09Z\tBACKUP-FAILED-UNITS-PREFLIGHT-221\ta2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8\t0\tOK\tcluster-audit\n=== EXACT_IMMUTABLE_RESULT_221 ===\nRESULT_221=MISSING_LOCAL\n=== CURRENT_FAILED_UNITS ===\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nhomelab-cluster-admin-pull.service    loaded failed failed Pull cluster-admin observer health into pve01\nhomelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03\nCHANGES_MADE=NO\n"
 }

runtime/latest.txt

@@ -1,5 +1,5 @@
 CHAT_OUTPUT_BEGIN
-COMMAND_ID=BACKUP-FAILED-UNITS-PREFLIGHT-221
+COMMAND_ID=BACKUP-FAILED-UNITS-RESULT-222
 STATUS=OK
 RC=0
 HOST=pve01
@@ -9,43 +9,25 @@ REFERENCE_REGISTER_CHECK=OK
 REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
 ERROR_REGISTER_CHECK=OK
 ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
-COMMAND_SHA256=a2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8
+COMMAND_SHA256=c1f54adfcc646548cd97343aa00aa80749ea640f6d3a077cebc664e582d3e591
 DUPLICATE_FAILED_COMMAND_BLOCKED=false
 EXECUTION_STARTED=true
 CHANGES_MADE=false
 SANITIZED=yes
 SECRETS_INCLUDED=no
 PRIVATE_ADDRESSES_INCLUDED=no
-RAW_EVIDENCE_SHA256=45b89b2bddc0a8064033309702ebcffe8b64609b2786bd3a6f4e0532491ac5e1
+RAW_EVIDENCE_SHA256=f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23
-SANITIZED_OUTPUT_SHA256=45b89b2bddc0a8064033309702ebcffe8b64609b2786bd3a6f4e0532491ac5e1
+SANITIZED_OUTPUT_SHA256=f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23
 OUTPUT_BEGIN
-PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220
+PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-PREFLIGHT-221
-=== LEDGER_219_220 ===
+=== LEDGER_221 ===
-2026-07-22T15:42:52Z	BACKUP-FAILED-UNITS-EVIDENCE-219	250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2	0	OK	cluster-audit
+2026-07-22T15:50:09Z	BACKUP-FAILED-UNITS-PREFLIGHT-221	a2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8	0	OK	cluster-audit
-2026-07-22T15:46:24Z	BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220	728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea	0	OK	cluster-audit
+=== EXACT_IMMUTABLE_RESULT_221 ===
-=== MATRIX_NON_OK ===
+RESULT_221=MISSING_LOCAL
-filebrowser_restore_validate	filebrowser-restore-validate	STALE	enabled	active	success	0	OK	1103338	864000	STALE	/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt
+=== CURRENT_FAILED_UNITS ===
-homelab_backup_sla_health	homelab-backup-sla-health	FAIL	enabled	active	success	0	WARN	658	172800	FRESH	/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt
+homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state
-homelab_mail_cloud_restore_drill	homelab-mail-cloud-restore-drill	STALE	enabled	active	success	0	OK	1075183	864000	STALE	/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt
+homelab-cluster-admin-pull.service    loaded failed failed Pull cluster-admin observer health into pve01
-netbird_vps_backup	netbird-vps-backup	FAIL	disabled	inactive	success	0	OK	1834810	864000	STALE	/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt
+homelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03
-paperless_restore_dry_run	paperless-restore-dry-run	STALE	enabled	active	success	0	OK	1088900	864000	STALE	/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt
-vm160_remote_backup	vm160-remote-backup	FAIL	disabled	inactive	exit-code	1	FAIL	113521	172800	FRESH	/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt
-=== VM160_PREFLIGHT ===
-HOST_CHECK_RC=0
-TARGET_WORKER_RC=0
-SOURCE_VM_RC=0
-status: running
-SOURCE_ACTIVE_RC=1
-SOURCE_IDLE_RC=0
-TARGET_FREE_QUERY_RC=0
-TARGET_FREE_BYTES=92632272896
-LATEST_SIZE_QUERY_RC=0
-LATEST_SIZE_BYTES=84995216465
-REQUIRED_BYTES=116981438821
-CAPACITY_CHECK_RC=1
-=== CLUSTER_ADMIN_PROBE_FAILURES ===
-/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
-/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
 CHANGES_MADE=NO
 
 OUTPUT_END