runtime: publish BACKUP-FAILED-UNITS-EVIDENCE-219
This commit is contained in:
homelab-runtime-publisher
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"channel": "homelab-runtime",
|
||||
"command_id": "BACKUP-FAILED-UNITS-EVIDENCE-219",
|
||||
"status": "OK",
|
||||
"rc": 0,
|
||||
"host": "pve01",
|
||||
"mode": "read-only",
|
||||
"component": "cluster-audit",
|
||||
"started_at_utc": "2026-07-22T15:42:52Z",
|
||||
"finished_at_utc": "2026-07-22T15:42:52Z",
|
||||
"reference_register_checked": true,
|
||||
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
|
||||
"error_register_checked": true,
|
||||
"error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
|
||||
"command_sha256": "250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2",
|
||||
"duplicate_failed_command_blocked": false,
|
||||
"block_reason": null,
|
||||
"execution_started": true,
|
||||
"changes_made": false,
|
||||
"sanitized": true,
|
||||
"secrets_included": false,
|
||||
"private_addresses_included": false,
|
||||
"raw_evidence_retained_locally": true,
|
||||
"raw_evidence_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607",
|
||||
"sanitized_output_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607",
|
||||
"output_truncated_in_json": false,
|
||||
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
||||
"output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218\n===LEDGER_218===\n2026-07-22T15:39:09Z\tBACKUP-FAILED-UNITS-DECISION-218\t67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730\t0\tOK\tcluster-audit\n===MATRIX_NON_OK===\n{\"id\":\"filebrowser_restore_validate\",\"name\":\"filebrowser-restore-validate\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"filebrowser-restore-validate.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\"}\n{\"id\":\"homelab_backup_sla_health\",\"name\":\"homelab-backup-sla-health\",\"status\":\"FAIL\",\"source_status\":\"WARN\",\"freshness\":\"FRESH\",\"service\":\"homelab-backup-sla-health.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\"}\n{\"id\":\"homelab_mail_cloud_restore_drill\",\"name\":\"homelab-mail-cloud-restore-drill\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"homelab-mail-cloud-restore-drill.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\"}\n{\"id\":\"netbird_vps_backup\",\"name\":\"netbird-vps-backup\",\"status\":\"FAIL\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"netbird-vps-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\"}\n{\"id\":\"paperless_restore_dry_run\",\"name\":\"paperless-restore-dry-run\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"paperless-restore-dry-run.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\"}\n{\"id\":\"vm160_remote_backup\",\"name\":\"vm160-remote-backup\",\"status\":\"FAIL\",\"source_status\":\"FAIL\",\"freshness\":\"FRESH\",\"service\":\"homelab-vm160-remote-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\"}\n===VM160_HEALTH===\nSTATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=20260720T233913Z\nVMID=160\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z\nRC=1\n===CLUSTER_ADMIN_FAILURES===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n"
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=cluster-audit
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||
COMMAND_SHA256=250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGES_MADE=false
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607
|
||||
SANITIZED_OUTPUT_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607
|
||||
OUTPUT_BEGIN
|
||||
PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218
|
||||
===LEDGER_218===
|
||||
2026-07-22T15:39:09Z BACKUP-FAILED-UNITS-DECISION-218 67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730 0 OK cluster-audit
|
||||
===MATRIX_NON_OK===
|
||||
{"id":"filebrowser_restore_validate","name":"filebrowser-restore-validate","status":"STALE","source_status":"OK","freshness":"STALE","service":"filebrowser-restore-validate.service","source_health":"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt"}
|
||||
{"id":"homelab_backup_sla_health","name":"homelab-backup-sla-health","status":"FAIL","source_status":"WARN","freshness":"FRESH","service":"homelab-backup-sla-health.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt"}
|
||||
{"id":"homelab_mail_cloud_restore_drill","name":"homelab-mail-cloud-restore-drill","status":"STALE","source_status":"OK","freshness":"STALE","service":"homelab-mail-cloud-restore-drill.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt"}
|
||||
{"id":"netbird_vps_backup","name":"netbird-vps-backup","status":"FAIL","source_status":"OK","freshness":"STALE","service":"netbird-vps-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt"}
|
||||
{"id":"paperless_restore_dry_run","name":"paperless-restore-dry-run","status":"STALE","source_status":"OK","freshness":"STALE","service":"paperless-restore-dry-run.service","source_health":"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt"}
|
||||
{"id":"vm160_remote_backup","name":"vm160-remote-backup","status":"FAIL","source_status":"FAIL","freshness":"FRESH","service":"homelab-vm160-remote-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt"}
|
||||
===VM160_HEALTH===
|
||||
STATUS=FAIL
|
||||
SERVICE=homelab-vm160-remote-backup
|
||||
TIMESTAMP_UTC=20260720T233913Z
|
||||
VMID=160
|
||||
TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z
|
||||
RC=1
|
||||
===CLUSTER_ADMIN_FAILURES===
|
||||
/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
|
||||
/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
|
||||
CHANGES_MADE=NO
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+7
-7
@@ -1,19 +1,19 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"channel": "homelab-runtime",
|
||||
"command_id": "BACKUP-FAILED-UNITS-DECISION-218",
|
||||
"command_id": "BACKUP-FAILED-UNITS-EVIDENCE-219",
|
||||
"status": "OK",
|
||||
"rc": 0,
|
||||
"host": "pve01",
|
||||
"mode": "read-only",
|
||||
"component": "cluster-audit",
|
||||
"started_at_utc": "2026-07-22T15:39:09Z",
|
||||
"finished_at_utc": "2026-07-22T15:39:09Z",
|
||||
"started_at_utc": "2026-07-22T15:42:52Z",
|
||||
"finished_at_utc": "2026-07-22T15:42:52Z",
|
||||
"reference_register_checked": true,
|
||||
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
|
||||
"error_register_checked": true,
|
||||
"error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
|
||||
"command_sha256": "67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730",
|
||||
"command_sha256": "250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2",
|
||||
"duplicate_failed_command_blocked": false,
|
||||
"block_reason": null,
|
||||
"execution_started": true,
|
||||
@@ -22,9 +22,9 @@
|
||||
"secrets_included": false,
|
||||
"private_addresses_included": false,
|
||||
"raw_evidence_retained_locally": true,
|
||||
"raw_evidence_sha256": "ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1",
|
||||
"sanitized_output_sha256": "d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733",
|
||||
"raw_evidence_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607",
|
||||
"sanitized_output_sha256": "5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607",
|
||||
"output_truncated_in_json": false,
|
||||
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
||||
"output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217\n=== LEDGER_217 ===\n2026-07-22T15:35:58Z\tBACKUP-FAILED-UNITS-SOURCE-217\t2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86\t0\tOK\tcluster-audit\n=== MATRIX_ENTRYPOINT ===\n 1\t#!/usr/bin/env python3\n 2\tfrom pathlib import Path\n 3\timport shutil\n 4\timport json,subprocess,os,time,sys,hashlib\n 5\tM=Path(\"/etc/homelab-backup-matrix/matrix.json\")\n 6\tROOT=Path(\"/var/lib/homelab-backup-matrix\")\n 7\tRENDER=ROOT/\"rendered\"\n 8\tCOMPAT=ROOT/\"compat\"\n 9\tdef cmd(args):\n 10\t p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)\n 11\t return p.stdout.strip() or \"unknown\"\n 12\tdef atomic(path,text,mode=0o640):\n 13\t path.parent.mkdir(parents=True,exist_ok=True)\n 14\t tmp=path.with_name(path.name+\".tmp\")\n 15\t tmp.write_text(text,encoding=\"utf-8\")\n 16\t os.chmod(tmp,mode)\n 17\t os.replace(tmp,path)\n 18\tdef parse_health(path):\n=== VM160_CONTROL_FLOW ===\n5:SOURCE=root@[PRIVATE_IP]\n6:TARGET=root@[PRIVATE_IP]\n7:KH=[SENSITIVE_PATH]\n8:PARENT=/mnt/staging/vzdump\n11:LOCK=/run/lock/homelab-vm160-remote-backup.lock\n13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker\n14:target_worker(){ ssh \"${SSH[@]}\" \"$TARGET\" bash -s -- \"$@\" < \"$TARGET_WORKER\"; }\n15:install -d -m 0755 \"$(dirname \"$HEALTH\")\" \"$(dirname \"$LOCK\")\"\n16:exec 9>\"$LOCK\"\n19:test -x \"$TARGET_WORKER\"\n20:test \"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"qm status $VMID\" | awk '{print $2}')\" = running\n21:SOURCE_ACTIVE_RC=0\n22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n23:test \"$SOURCE_ACTIVE_RC\" -eq 0 || test \"$SOURCE_ACTIVE_RC\" -eq 1\n24:test -z \"$SOURCE_ACTIVE\"\n27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n35: ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n37: printf 'STATUS=FAIL\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nTARGET_DIR=%s\\nRC=%s\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$rc\" > \"$tmp\"\n44:trap cleanup EXIT\n45:REMOTE_FREE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"df -B1 --output=avail '$PARENT' | awk 'NR==2{print \\$1}'\")\"\n46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n48:if test \"$LATEST_SIZE\" -gt 0; then\n50:else\n52:fi\n53:test \"$REMOTE_FREE\" -ge \"$REQUIRED\"\n54:ssh \"${SSH[@]}\" \"$TARGET\" \"install -d -m 0700 '$DIR'\"\n55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n58:SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"stat -c %s '$FINAL'\")\"\n59:SHA=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"sha256sum '$FINAL' | awk '{print \\$1}'\")\"\n62:printf 'STATUS=OK\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nSOURCE_NODE=pve02\\nTARGET_NODE=pve03\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\nREMOTE_RETENTION_GENERATIONS=1\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\" > \"$tmp\"\n65:trap - EXIT\n67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\n' \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\"\n=== CLUSTER_ADMIN_CURRENT_FAILURE ===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n"
|
||||
"output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218\n===LEDGER_218===\n2026-07-22T15:39:09Z\tBACKUP-FAILED-UNITS-DECISION-218\t67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730\t0\tOK\tcluster-audit\n===MATRIX_NON_OK===\n{\"id\":\"filebrowser_restore_validate\",\"name\":\"filebrowser-restore-validate\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"filebrowser-restore-validate.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt\"}\n{\"id\":\"homelab_backup_sla_health\",\"name\":\"homelab-backup-sla-health\",\"status\":\"FAIL\",\"source_status\":\"WARN\",\"freshness\":\"FRESH\",\"service\":\"homelab-backup-sla-health.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt\"}\n{\"id\":\"homelab_mail_cloud_restore_drill\",\"name\":\"homelab-mail-cloud-restore-drill\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"homelab-mail-cloud-restore-drill.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt\"}\n{\"id\":\"netbird_vps_backup\",\"name\":\"netbird-vps-backup\",\"status\":\"FAIL\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"netbird-vps-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt\"}\n{\"id\":\"paperless_restore_dry_run\",\"name\":\"paperless-restore-dry-run\",\"status\":\"STALE\",\"source_status\":\"OK\",\"freshness\":\"STALE\",\"service\":\"paperless-restore-dry-run.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt\"}\n{\"id\":\"vm160_remote_backup\",\"name\":\"vm160-remote-backup\",\"status\":\"FAIL\",\"source_status\":\"FAIL\",\"freshness\":\"FRESH\",\"service\":\"homelab-vm160-remote-backup.service\",\"source_health\":\"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt\"}\n===VM160_HEALTH===\nSTATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=20260720T233913Z\nVMID=160\nTARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z\nRC=1\n===CLUSTER_ADMIN_FAILURES===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n"
|
||||
}
|
||||
|
||||
+22
-61
@@ -1,5 +1,5 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218
|
||||
COMMAND_ID=BACKUP-FAILED-UNITS-EVIDENCE-219
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
@@ -9,73 +9,34 @@ REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||
COMMAND_SHA256=67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730
|
||||
COMMAND_SHA256=250df985b093bf4cef1ec9ae0d4853f332e3304de3a397989de4a4152481f4b2
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGES_MADE=false
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1
|
||||
SANITIZED_OUTPUT_SHA256=d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733
|
||||
RAW_EVIDENCE_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607
|
||||
SANITIZED_OUTPUT_SHA256=5f88fb28dcf9b5617175ba9b6f19b7e14fa3feb18e80e5e7bacabdcf9a1d6607
|
||||
OUTPUT_BEGIN
|
||||
PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217
|
||||
=== LEDGER_217 ===
|
||||
2026-07-22T15:35:58Z BACKUP-FAILED-UNITS-SOURCE-217 2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86 0 OK cluster-audit
|
||||
=== MATRIX_ENTRYPOINT ===
|
||||
1 #!/usr/bin/env python3
|
||||
2 from pathlib import Path
|
||||
3 import shutil
|
||||
4 import json,subprocess,os,time,sys,hashlib
|
||||
5 M=Path("/etc/homelab-backup-matrix/matrix.json")
|
||||
6 ROOT=Path("/var/lib/homelab-backup-matrix")
|
||||
7 RENDER=ROOT/"rendered"
|
||||
8 COMPAT=ROOT/"compat"
|
||||
9 def cmd(args):
|
||||
10 p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)
|
||||
11 return p.stdout.strip() or "unknown"
|
||||
12 def atomic(path,text,mode=0o640):
|
||||
13 path.parent.mkdir(parents=True,exist_ok=True)
|
||||
14 tmp=path.with_name(path.name+".tmp")
|
||||
15 tmp.write_text(text,encoding="utf-8")
|
||||
16 os.chmod(tmp,mode)
|
||||
17 os.replace(tmp,path)
|
||||
18 def parse_health(path):
|
||||
=== VM160_CONTROL_FLOW ===
|
||||
5:SOURCE=root@[PRIVATE_IP]
|
||||
6:TARGET=root@[PRIVATE_IP]
|
||||
7:KH=[SENSITIVE_PATH]
|
||||
8:PARENT=/mnt/staging/vzdump
|
||||
11:LOCK=/run/lock/homelab-vm160-remote-backup.lock
|
||||
13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker
|
||||
14:target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; }
|
||||
15:install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")"
|
||||
16:exec 9>"$LOCK"
|
||||
19:test -x "$TARGET_WORKER"
|
||||
20:test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running
|
||||
21:SOURCE_ACTIVE_RC=0
|
||||
22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$?
|
||||
23:test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1
|
||||
24:test -z "$SOURCE_ACTIVE"
|
||||
27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst"
|
||||
35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true
|
||||
37: printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp"
|
||||
44:trap cleanup EXIT
|
||||
45:REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")"
|
||||
46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")"
|
||||
48:if test "$LATEST_SIZE" -gt 0; then
|
||||
50:else
|
||||
52:fi
|
||||
53:test "$REMOTE_FREE" -ge "$REQUIRED"
|
||||
54:ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'"
|
||||
55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'"
|
||||
57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG"
|
||||
58:SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")"
|
||||
59:SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")"
|
||||
62:printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp"
|
||||
65:trap - EXIT
|
||||
67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA"
|
||||
=== CLUSTER_ADMIN_CURRENT_FAILURE ===
|
||||
PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218
|
||||
===LEDGER_218===
|
||||
2026-07-22T15:39:09Z BACKUP-FAILED-UNITS-DECISION-218 67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730 0 OK cluster-audit
|
||||
===MATRIX_NON_OK===
|
||||
{"id":"filebrowser_restore_validate","name":"filebrowser-restore-validate","status":"STALE","source_status":"OK","freshness":"STALE","service":"filebrowser-restore-validate.service","source_health":"/var/lib/homelab-backup-matrix/evidence/filebrowser_restore_validate/0-filebrowser-restore.txt"}
|
||||
{"id":"homelab_backup_sla_health","name":"homelab-backup-sla-health","status":"FAIL","source_status":"WARN","freshness":"FRESH","service":"homelab-backup-sla-health.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_backup_sla_health/0-backup-sla.txt"}
|
||||
{"id":"homelab_mail_cloud_restore_drill","name":"homelab-mail-cloud-restore-drill","status":"STALE","source_status":"OK","freshness":"STALE","service":"homelab-mail-cloud-restore-drill.service","source_health":"/var/lib/homelab-backup-matrix/evidence/homelab_mail_cloud_restore_drill/0-mail-cloud-restore-drill.txt"}
|
||||
{"id":"netbird_vps_backup","name":"netbird-vps-backup","status":"FAIL","source_status":"OK","freshness":"STALE","service":"netbird-vps-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/netbird_vps_backup/0-netbird-vps-backup.txt"}
|
||||
{"id":"paperless_restore_dry_run","name":"paperless-restore-dry-run","status":"STALE","source_status":"OK","freshness":"STALE","service":"paperless-restore-dry-run.service","source_health":"/var/lib/homelab-backup-matrix/evidence/paperless_restore_dry_run/0-paperless-restore.txt"}
|
||||
{"id":"vm160_remote_backup","name":"vm160-remote-backup","status":"FAIL","source_status":"FAIL","freshness":"FRESH","service":"homelab-vm160-remote-backup.service","source_health":"/var/lib/homelab-backup-matrix/evidence/vm160_remote_backup/0-vm160-remote-backup.txt"}
|
||||
===VM160_HEALTH===
|
||||
STATUS=FAIL
|
||||
SERVICE=homelab-vm160-remote-backup
|
||||
TIMESTAMP_UTC=20260720T233913Z
|
||||
VMID=160
|
||||
TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260720T233913Z
|
||||
RC=1
|
||||
===CLUSTER_ADMIN_FAILURES===
|
||||
/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
|
||||
/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
|
||||
CHANGES_MADE=NO
|
||||
|
||||
Reference in New Issue
Block a user