From 4154e95c8504e9972b1e78dcc3875e8c72062a5a Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Wed, 22 Jul 2026 15:46:24 +0000 Subject: [PATCH] runtime: publish BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220 --- ...ACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.json | 30 +++++++++++++++++ ...BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.txt | 30 +++++++++++++++++ runtime/latest.json | 14 ++++---- runtime/latest.txt | 33 +++++-------------- 4 files changed, 76 insertions(+), 31 deletions(-) create mode 100644 runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.json create mode 100644 runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.txt diff --git a/runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.json b/runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.json new file mode 100644 index 0000000..23a41ab --- /dev/null +++ b/runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-22T15:46:24Z", + "finished_at_utc": "2026-07-22T15:46:24Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331", + "sanitized_output_sha256": "7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219\n=== LEDGER_219 ===\n2026-07-22T15:42:52Z\tBACKUP-FAILED-UNITS-EVIDENCE-219\t250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2\t0\tOK\tcluster-audit\n=== IMMUTABLE_LOCAL_RESULT_219 ===\nLOCAL_RESULT_219=MISSING\nCHANGES_MADE=NO\n" +} diff --git a/runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.txt b/runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.txt new file mode 100644 index 0000000..117699c --- /dev/null +++ b/runtime/history/BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220.txt @@ -0,0 +1,30 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331 +SANITIZED_OUTPUT_SHA256=7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331 +OUTPUT_BEGIN +PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219 +=== LEDGER_219 === +2026-07-22T15:42:52Z BACKUP-FAILED-UNITS-EVIDENCE-219 250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2 0 OK cluster-audit +=== IMMUTABLE_LOCAL_RESULT_219 === +LOCAL_RESULT_219=MISSING +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index e52921a..23a41ab 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-FAILED-UNITS-EVIDENCE-219", + "command_id": "BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-22T15:42:52Z", - "finished_at_utc": "2026-07-22T15:42:52Z", + "started_at_utc": "2026-07-22T15:46:24Z", + "finished_at_utc": "2026-07-22T15:46:24Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2", + "command_sha256": "728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607", - "sanitized_output_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607", + "raw_evidence_sha256": "7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331", + "sanitized_output_sha256": "7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218\n===LEDGER_218===\n2026-07-22T15:39:09Z\tBACKUP-FAILED-UNITS-DECISION-218\t67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730\t0\tOK\tcluster-audit\n===MATRIX_NON_OK===\n{\"id\":\"filebrowser_restore_validate\",\"name\":\"filebrowser-restore-validate\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"filebrowser-restore-validate.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\"}\n{\"id\":\"homelab_backup_sla_health\",\"name\":\"homelab-backup-sla-health\",\"status\":\"FAIL\",\"source_status\":\"WARN\",\"freshness\":\"FRESH\",\"service\":\"homelab-backup-sla-health.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\"}\n{\"id\":\"homelab_mail_cloud_restore_drill\",\"name\":\"homelab-mail-cloud-restore-drill\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"homelab-mail-cloud-restore-drill.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\"}\n{\"id\":\"netbird_vps_backup\",\"name\":\"netbird-vps-backup\",\"status\":\"FAIL\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"netbird-vps-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\"}\n{\"id\":\"paperless_restore_dry_run\",\"name\":\"paperless-restore-dry-run\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"paperless-restore-dry-run.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\"}\n{\"id\":\"vm160_remote_backup\",\"name\":\"vm160-remote-backup\",\"status\":\"FAIL\",\"source_status\":\"FAIL\",\"freshness\":\"FRESH\",\"service\":\"homelab-vm160-remote-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\"}\n===VM160_HEALTH===\nSTATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=20260720T233913Z\nVMID=160\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z\nRC=1\n===CLUSTER_ADMIN_FAILURES===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n" + "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219\n=== LEDGER_219 ===\n2026-07-22T15:42:52Z\tBACKUP-FAILED-UNITS-EVIDENCE-219\t250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2\t0\tOK\tcluster-audit\n=== IMMUTABLE_LOCAL_RESULT_219 ===\nLOCAL_RESULT_219=MISSING\nCHANGES_MADE=NO\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 7cb5e4f..117699c 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219 +COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-RELAY-220 STATUS=OK RC=0 HOST=pve01 @@ -9,36 +9,21 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2 +COMMAND_SHA256=728e7b38b5216aa9c890b7e56909119839001945f6513c4e7427ac9a7078bbea DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607 -SANITIZED_OUTPUT_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607 +RAW_EVIDENCE_SHA256=7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331 +SANITIZED_OUTPUT_SHA256=7f06621eae3db375c1f0faccb3713a953e2c18ef961bc3e867e166364aa33331 OUTPUT_BEGIN -PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218 -===LEDGER_218=== -2026-07-22T15:39:09Z BACKUP-FAILED-UNITS-DECISION-218 67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730 0 OK cluster-audit -===MATRIX_NON_OK=== -{"id":"filebrowser_restore_validate","name":"filebrowser-restore-validate","status":"STALE","source_status":"OK","freshness":"STALE","service":"filebrowser-restore-validate.service","source_health":"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt"} -{"id":"homelab_backup_sla_health","name":"homelab-backup-sla-health","status":"FAIL","source_status":"WARN","freshness":"FRESH","service":"homelab-backup-sla-health.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt"} -{"id":"homelab_mail_cloud_restore_drill","name":"homelab-mail-cloud-restore-drill","status":"STALE","source_status":"OK","freshness":"STALE","service":"homelab-mail-cloud-restore-drill.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt"} -{"id":"netbird_vps_backup","name":"netbird-vps-backup","status":"FAIL","source_status":"OK","freshness":"STALE","service":"netbird-vps-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt"} -{"id":"paperless_restore_dry_run","name":"paperless-restore-dry-run","status":"STALE","source_status":"OK","freshness":"STALE","service":"paperless-restore-dry-run.service","source_health":"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt"} -{"id":"vm160_remote_backup","name":"vm160-remote-backup","status":"FAIL","source_status":"FAIL","freshness":"FRESH","service":"homelab-vm160-remote-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt"} -===VM160_HEALTH=== -STATUS=FAIL -SERVICE=homelab-vm160-remote-backup -TIMESTAMP_UTC=20260720T233913Z -VMID=160 -TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z -RC=1 -===CLUSTER_ADMIN_FAILURES=== -/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md -/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md +PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219 +=== LEDGER_219 === +2026-07-22T15:42:52Z BACKUP-FAILED-UNITS-EVIDENCE-219 250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2 0 OK cluster-audit +=== IMMUTABLE_LOCAL_RESULT_219 === +LOCAL_RESULT_219=MISSING CHANGES_MADE=NO OUTPUT_END