From 19485ec9bd231482342fd2f04e5df31c09853f25 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Wed, 22 Jul 2026 15:56:40 +0000 Subject: [PATCH] runtime: publish BACKUP-FAILED-UNITS-RESULT-222 --- .../BACKUP-FAILED-UNITS-RESULT-222.json | 30 +++++++++++++ .../BACKUP-FAILED-UNITS-RESULT-222.txt | 34 ++++++++++++++ runtime/latest.json | 14 +++--- runtime/latest.txt | 44 ++++++------------- 4 files changed, 84 insertions(+), 38 deletions(-) create mode 100644 runtime/history/BACKUP-FAILED-UNITS-RESULT-222.json create mode 100644 runtime/history/BACKUP-FAILED-UNITS-RESULT-222.txt diff --git a/runtime/history/BACKUP-FAILED-UNITS-RESULT-222.json b/runtime/history/BACKUP-FAILED-UNITS-RESULT-222.json new file mode 100644 index 0000000..25f7c13 --- /dev/null +++ b/runtime/history/BACKUP-FAILED-UNITS-RESULT-222.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-FAILED-UNITS-RESULT-222", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-22T15:56:39Z", + "finished_at_utc": "2026-07-22T15:56:39Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "c1f54adfcc646548cd97343aa00aa80749ea640f6d3a077cebc664e582d3e591", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23", + "sanitized_output_sha256": "f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-PREFLIGHT-221\n=== LEDGER_221 ===\n2026-07-22T15:50:09Z\tBACKUP-FAILED-UNITS-PREFLIGHT-221\ta2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8\t0\tOK\tcluster-audit\n=== EXACT_IMMUTABLE_RESULT_221 ===\nRESULT_221=MISSING_LOCAL\n=== CURRENT_FAILED_UNITS ===\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nhomelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01\nhomelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\nCHANGES_MADE=NO\n" +} diff --git a/runtime/history/BACKUP-FAILED-UNITS-RESULT-222.txt b/runtime/history/BACKUP-FAILED-UNITS-RESULT-222.txt new file mode 100644 index 0000000..4713f21 --- /dev/null +++ b/runtime/history/BACKUP-FAILED-UNITS-RESULT-222.txt @@ -0,0 +1,34 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-FAILED-UNITS-RESULT-222 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=c1f54adfcc646548cd97343aa00aa80749ea640f6d3a077cebc664e582d3e591 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23 +SANITIZED_OUTPUT_SHA256=f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23 +OUTPUT_BEGIN +PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-PREFLIGHT-221 +=== LEDGER_221 === +2026-07-22T15:50:09Z BACKUP-FAILED-UNITS-PREFLIGHT-221 a2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8 0 OK cluster-audit +=== EXACT_IMMUTABLE_RESULT_221 === +RESULT_221=MISSING_LOCAL +=== CURRENT_FAILED_UNITS === +homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state +homelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01 +homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index a98ecd9..25f7c13 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-FAILED-UNITS-PREFLIGHT-221", + "command_id": "BACKUP-FAILED-UNITS-RESULT-222", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-22T15:50:08Z", - "finished_at_utc": "2026-07-22T15:50:09Z", + "started_at_utc": "2026-07-22T15:56:39Z", + "finished_at_utc": "2026-07-22T15:56:39Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "a2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8", + "command_sha256": "c1f54adfcc646548cd97343aa00aa80749ea640f6d3a077cebc664e582d3e591", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "45b89b2bddc0a8064033309702ebcffe8b64609b2786bd3a6f4e0532491ac5e1", - "sanitized_output_sha256": "45b89b2bddc0a8064033309702ebcffe8b64609b2786bd3a6f4e0532491ac5e1", + "raw_evidence_sha256": "f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23", + "sanitized_output_sha256": "f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220\n=== LEDGER_219_220 ===\n2026-07-22T15:42:52Z\tBACKUP-FAILED-UNITS-EVIDENCE-219\t250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2\t0\tOK\tcluster-audit\n2026-07-22T15:46:24Z\tBACKUP-FAILED-UNITS-EVIDENCE-RELAY-220\t728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea\t0\tOK\tcluster-audit\n=== MATRIX_NON_OK ===\nfilebrowser_restore_validate\tfilebrowser-restore-validate\tSTALE\tenabled\tactive\tsuccess\t0\tOK\t1103338\t864000\tSTALE\t/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\nhomelab_backup_sla_health\thomelab-backup-sla-health\tFAIL\tenabled\tactive\tsuccess\t0\tWARN\t658\t172800\tFRESH\t/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\nhomelab_mail_cloud_restore_drill\thomelab-mail-cloud-restore-drill\tSTALE\tenabled\tactive\tsuccess\t0\tOK\t1075183\t864000\tSTALE\t/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\nnetbird_vps_backup\tnetbird-vps-backup\tFAIL\tdisabled\tinactive\tsuccess\t0\tOK\t1834810\t864000\tSTALE\t/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\npaperless_restore_dry_run\tpaperless-restore-dry-run\tSTALE\tenabled\tactive\tsuccess\t0\tOK\t1088900\t864000\tSTALE\t/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\nvm160_remote_backup\tvm160-remote-backup\tFAIL\tdisabled\tinactive\texit-code\t1\tFAIL\t113521\t172800\tFRESH\t/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\n=== VM160_PREFLIGHT ===\nHOST_CHECK_RC=0\nTARGET_WORKER_RC=0\nSOURCE_VM_RC=0\nstatus: running\nSOURCE_ACTIVE_RC=1\nSOURCE_IDLE_RC=0\nTARGET_FREE_QUERY_RC=0\nTARGET_FREE_BYTES=92632272896\nLATEST_SIZE_QUERY_RC=0\nLATEST_SIZE_BYTES=84995216465\nREQUIRED_BYTES=116981438821\nCAPACITY_CHECK_RC=1\n=== CLUSTER_ADMIN_PROBE_FAILURES ===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n" + "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-PREFLIGHT-221\n=== LEDGER_221 ===\n2026-07-22T15:50:09Z\tBACKUP-FAILED-UNITS-PREFLIGHT-221\ta2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8\t0\tOK\tcluster-audit\n=== EXACT_IMMUTABLE_RESULT_221 ===\nRESULT_221=MISSING_LOCAL\n=== CURRENT_FAILED_UNITS ===\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nhomelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01\nhomelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\nCHANGES_MADE=NO\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 4f705bc..4713f21 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-FAILED-UNITS-PREFLIGHT-221 +COMMAND_ID=BACKUP-FAILED-UNITS-RESULT-222 STATUS=OK RC=0 HOST=pve01 @@ -9,43 +9,25 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=a2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8 +COMMAND_SHA256=c1f54adfcc646548cd97343aa00aa80749ea640f6d3a077cebc664e582d3e591 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=45b89b2bddc0a8064033309702ebcffe8b64609b2786bd3a6f4e0532491ac5e1 -SANITIZED_OUTPUT_SHA256=45b89b2bddc0a8064033309702ebcffe8b64609b2786bd3a6f4e0532491ac5e1 +RAW_EVIDENCE_SHA256=f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23 +SANITIZED_OUTPUT_SHA256=f4cfe3fb96c95b281990310645aae310dae80ba1d8258f2c436694c73e5dbe23 OUTPUT_BEGIN -PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220 -=== LEDGER_219_220 === -2026-07-22T15:42:52Z BACKUP-FAILED-UNITS-EVIDENCE-219 250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2 0 OK cluster-audit -2026-07-22T15:46:24Z BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220 728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea 0 OK cluster-audit -=== MATRIX_NON_OK === -filebrowser_restore_validate filebrowser-restore-validate STALE enabled active success 0 OK 1103338 864000 STALE /var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt -homelab_backup_sla_health homelab-backup-sla-health FAIL enabled active success 0 WARN 658 172800 FRESH /var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt -homelab_mail_cloud_restore_drill homelab-mail-cloud-restore-drill STALE enabled active success 0 OK 1075183 864000 STALE /var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt -netbird_vps_backup netbird-vps-backup FAIL disabled inactive success 0 OK 1834810 864000 STALE /var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt -paperless_restore_dry_run paperless-restore-dry-run STALE enabled active success 0 OK 1088900 864000 STALE /var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt -vm160_remote_backup vm160-remote-backup FAIL disabled inactive exit-code 1 FAIL 113521 172800 FRESH /var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt -=== VM160_PREFLIGHT === -HOST_CHECK_RC=0 -TARGET_WORKER_RC=0 -SOURCE_VM_RC=0 -status: running -SOURCE_ACTIVE_RC=1 -SOURCE_IDLE_RC=0 -TARGET_FREE_QUERY_RC=0 -TARGET_FREE_BYTES=92632272896 -LATEST_SIZE_QUERY_RC=0 -LATEST_SIZE_BYTES=84995216465 -REQUIRED_BYTES=116981438821 -CAPACITY_CHECK_RC=1 -=== CLUSTER_ADMIN_PROBE_FAILURES === -/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md -/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md +PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-PREFLIGHT-221 +=== LEDGER_221 === +2026-07-22T15:50:09Z BACKUP-FAILED-UNITS-PREFLIGHT-221 a2ec12029931b3f9df2ff411cc892623068c83fc30c2b899b87bc636ca2e09f8 0 OK cluster-audit +=== EXACT_IMMUTABLE_RESULT_221 === +RESULT_221=MISSING_LOCAL +=== CURRENT_FAILED_UNITS === +homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state +homelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01 +homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 CHANGES_MADE=NO OUTPUT_END