diff --git a/runtime/history/BACKUP-FAILED-UNITS-DECISION-218.json b/runtime/history/BACKUP-FAILED-UNITS-DECISION-218.json new file mode 100644 index 0000000..88fbe57 --- /dev/null +++ b/runtime/history/BACKUP-FAILED-UNITS-DECISION-218.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-FAILED-UNITS-DECISION-218", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-22T15:39:09Z", + "finished_at_utc": "2026-07-22T15:39:09Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1", + "sanitized_output_sha256": "d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217\n=== LEDGER_217 ===\n2026-07-22T15:35:58Z\tBACKUP-FAILED-UNITS-SOURCE-217\t2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86\t0\tOK\tcluster-audit\n=== MATRIX_ENTRYPOINT ===\n 1\t#!/usr/bin/env python3\n 2\tfrom pathlib import Path\n 3\timport shutil\n 4\timport json,subprocess,os,time,sys,hashlib\n 5\tM=Path(\"/etc/homelab-backup-matrix/matrix.json\")\n 6\tROOT=Path(\"/var/lib/homelab-backup-matrix\")\n 7\tRENDER=ROOT/\"rendered\"\n 8\tCOMPAT=ROOT/\"compat\"\n 9\tdef cmd(args):\n 10\t p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)\n 11\t return p.stdout.strip() or \"unknown\"\n 12\tdef atomic(path,text,mode=0o640):\n 13\t path.parent.mkdir(parents=True,exist_ok=True)\n 14\t tmp=path.with_name(path.name+\".tmp\")\n 15\t tmp.write_text(text,encoding=\"utf-8\")\n 16\t os.chmod(tmp,mode)\n 17\t os.replace(tmp,path)\n 18\tdef parse_health(path):\n=== VM160_CONTROL_FLOW ===\n5:SOURCE=root@[PRIVATE_IP]\n6:TARGET=root@[PRIVATE_IP]\n7:KH=[SENSITIVE_PATH]\n8:PARENT=/mnt/staging/vzdump\n11:LOCK=/run/lock/homelab-vm160-remote-backup.lock\n13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker\n14:target_worker(){ ssh \"${SSH[@]}\" \"$TARGET\" bash -s -- \"$@\" < \"$TARGET_WORKER\"; }\n15:install -d -m 0755 \"$(dirname \"$HEALTH\")\" \"$(dirname \"$LOCK\")\"\n16:exec 9>\"$LOCK\"\n19:test -x \"$TARGET_WORKER\"\n20:test \"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"qm status $VMID\" | awk '{print $2}')\" = running\n21:SOURCE_ACTIVE_RC=0\n22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n23:test \"$SOURCE_ACTIVE_RC\" -eq 0 || test \"$SOURCE_ACTIVE_RC\" -eq 1\n24:test -z \"$SOURCE_ACTIVE\"\n27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n35: ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n37: printf 'STATUS=FAIL\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nTARGET_DIR=%s\\nRC=%s\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$rc\" > \"$tmp\"\n44:trap cleanup EXIT\n45:REMOTE_FREE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"df -B1 --output=avail '$PARENT' | awk 'NR==2{print \\$1}'\")\"\n46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n48:if test \"$LATEST_SIZE\" -gt 0; then\n50:else\n52:fi\n53:test \"$REMOTE_FREE\" -ge \"$REQUIRED\"\n54:ssh \"${SSH[@]}\" \"$TARGET\" \"install -d -m 0700 '$DIR'\"\n55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n58:SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"stat -c %s '$FINAL'\")\"\n59:SHA=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"sha256sum '$FINAL' | awk '{print \\$1}'\")\"\n62:printf 'STATUS=OK\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nSOURCE_NODE=pve02\\nTARGET_NODE=pve03\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\nREMOTE_RETENTION_GENERATIONS=1\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\" > \"$tmp\"\n65:trap - EXIT\n67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\n' \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\"\n=== CLUSTER_ADMIN_CURRENT_FAILURE ===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n" +} diff --git a/runtime/history/BACKUP-FAILED-UNITS-DECISION-218.txt b/runtime/history/BACKUP-FAILED-UNITS-DECISION-218.txt new file mode 100644 index 0000000..69b65a0 --- /dev/null +++ b/runtime/history/BACKUP-FAILED-UNITS-DECISION-218.txt @@ -0,0 +1,84 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1 +SANITIZED_OUTPUT_SHA256=d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733 +OUTPUT_BEGIN +PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217 +=== LEDGER_217 === +2026-07-22T15:35:58Z BACKUP-FAILED-UNITS-SOURCE-217 2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86 0 OK cluster-audit +=== MATRIX_ENTRYPOINT === + 1 #!/usr/bin/env python3 + 2 from pathlib import Path + 3 import shutil + 4 import json,subprocess,os,time,sys,hashlib + 5 M=Path("/etc/homelab-backup-matrix/matrix.json") + 6 ROOT=Path("/var/lib/homelab-backup-matrix") + 7 RENDER=ROOT/"rendered" + 8 COMPAT=ROOT/"compat" + 9 def cmd(args): + 10 p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False) + 11 return p.stdout.strip() or "unknown" + 12 def atomic(path,text,mode=0o640): + 13 path.parent.mkdir(parents=True,exist_ok=True) + 14 tmp=path.with_name(path.name+".tmp") + 15 tmp.write_text(text,encoding="utf-8") + 16 os.chmod(tmp,mode) + 17 os.replace(tmp,path) + 18 def parse_health(path): +=== VM160_CONTROL_FLOW === +5:SOURCE=root@[PRIVATE_IP] +6:TARGET=root@[PRIVATE_IP] +7:KH=[SENSITIVE_PATH] +8:PARENT=/mnt/staging/vzdump +11:LOCK=/run/lock/homelab-vm160-remote-backup.lock +13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker +14:target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; } +15:install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")" +16:exec 9>"$LOCK" +19:test -x "$TARGET_WORKER" +20:test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running +21:SOURCE_ACTIVE_RC=0 +22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$? +23:test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1 +24:test -z "$SOURCE_ACTIVE" +27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst" +35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true +37: printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp" +44:trap cleanup EXIT +45:REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")" +46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")" +48:if test "$LATEST_SIZE" -gt 0; then +50:else +52:fi +53:test "$REMOTE_FREE" -ge "$REQUIRED" +54:ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'" +55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" +57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" +58:SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")" +59:SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")" +62:printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp" +65:trap - EXIT +67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA" +=== CLUSTER_ADMIN_CURRENT_FAILURE === +/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md +/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 96a401e..88fbe57 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-FAILED-UNITS-SOURCE-217", + "command_id": "BACKUP-FAILED-UNITS-DECISION-218", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-22T15:35:58Z", - "finished_at_utc": "2026-07-22T15:35:58Z", + "started_at_utc": "2026-07-22T15:39:09Z", + "finished_at_utc": "2026-07-22T15:39:09Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86", + "command_sha256": "67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "96e72a8cec4b5e5cde52a48e6816f4ce162311b07530ee398c57efc67f3480b1", - "sanitized_output_sha256": "dbe6bb8b0419312d9774755ff8668c435118bc102b49b0255667d04f06dda499", + "raw_evidence_sha256": "ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1", + "sanitized_output_sha256": "d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "PREVIOUS_COMMAND_ID=RUNTIME-PUBLISH-GAP-DIAG-216\n===== UNIT homelab-backup-matrix-refresh =====\n# /etc/systemd/system/homelab-backup-matrix-refresh.service\n[Unit]\nDescription=Generate unified homelab backup matrix state\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-matrix-refresh\nTimeoutStartSec=5min\n===== UNIT homelab-vm160-remote-backup =====\n# /etc/systemd/system/homelab-vm160-remote-backup.service\n[Unit]\nDescription=VM160 full-image backup streamed from pve02 to pve03\nAfter=network-online.target pve-cluster.service\nWants=network-online.target\n\n[Service]\nType=oneshot\nTimeoutStartSec=6h\nExecStart=/usr/local/sbin/homelab-vm160-remote-backup\n===== UNIT homelab-cluster-admin-pull =====\n# /etc/systemd/system/homelab-cluster-admin-pull.service\n[Unit]\nDescription=Pull cluster-admin observer health into pve01\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-cluster-admin-pull\n===== FILE /usr/local/sbin/homelab-backup-matrix-refresh =====\n/usr/local/sbin/homelab-backup-matrix-refresh: Python script, ASCII text executable, with very long lines (592)\n 1\t#!/usr/bin/env python3\n 2\tfrom pathlib import Path\n 3\timport shutil\n 4\timport json,subprocess,os,time,sys,hashlib\n 5\tM=Path(\"/etc/homelab-backup-matrix/matrix.json\")\n 6\tROOT=Path(\"/var/lib/homelab-backup-matrix\")\n 7\tRENDER=ROOT/\"rendered\"\n 8\tCOMPAT=ROOT/\"compat\"\n 9\tdef cmd(args):\n 10\t p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)\n 11\t return p.stdout.strip() or \"unknown\"\n 12\tdef atomic(path,text,mode=0o640):\n 13\t path.parent.mkdir(parents=True,exist_ok=True)\n 14\t tmp=path.with_name(path.name+\".tmp\")\n 15\t tmp.write_text(text,encoding=\"utf-8\")\n 16\t os.chmod(tmp,mode)\n 17\t os.replace(tmp,path)\n 18\tdef parse_health(path):\n 19\t if path.suffix.lower()==\".json\":\n 20\t try:\n 21\t data=json.loads(path.read_text(encoding=\"utf-8\",errors=\"replace\"))\n 22\t return str(data.get(\"status\",data.get(\"STATUS\",data.get(\"result\",\"UNKNOWN\"))))\n 23\t except Exception:\n 24\t return \"ERROR\"\n 25\t data={}\n 26\t text=path.read_text(encoding=\"utf-8\",errors=\"replace\")\n 27\t for token in text.replace(chr(10),\" \").split():\n 28\t if \"=\" in token:\n 29\t key,value=token.split(\"=\",1)\n 30\t if key and key[0].isalpha():\n 31\t data[key]=value\n 32\t return str(data.get(\"STATUS\",data.get(\"status\",\"UNKNOWN\")))\n 33\tdef normalize(value):\n 34\t value=str(value).strip().upper()\n 35\t if value in {\"OK\",\"PASS\",\"PASSED\",\"SUCCESS\",\"HEALTHY\"}:\n 36\t return \"OK\"\n 37\t if value in {\"WARN\",\"WARNING\"}:\n 38\t return \"WARN\"\n 39\t if value in {\"FAIL\",\"FAILED\",\"ERROR\",\"CRITICAL\",\"BAD\"}:\n 40\t return \"FAIL\"\n 41\t return value or \"UNKNOWN\"\n 42\tmatrix=json.loads(M.read_text(encoding=\"utf-8\"))\n 43\tROOT.mkdir(parents=True,exist_ok=True)\n 44\tRENDER.mkdir(parents=True,exist_ok=True)\n 45\tCOMPAT.mkdir(parents=True,exist_ok=True)\n 46\tnow=int(time.time())\n 47\tnow_iso=time.strftime(\"%Y-%m-%dT%H:%M:%SZ\",time.gmtime(now))\n 48\tresults=[]\n 49\tcompat_map=[]\n 50\tfor job in matrix.get(\"jobs\",[]):\n 51\t original_sources=list(job.get(\"evidence_sources_original\",[]))\n 52\t evidence_inputs=list(job.get(\"evidence_inputs\",[]))\n 53\t if original_sources and len(original_sources)!=len(evidence_inputs):\n 54\t raise RuntimeError(\"evidence_mapping_length_mismatch:\"+str(job.get(\"name\",\"\")))\n 55\t for source_raw,destination_raw in zip(original_sources,evidence_inputs):\n 56\t source_path=Path(source_raw)\n 57\t destination_path=Path(destination_raw)\n 58\t if not source_path.is_file():\n 59\t continue\n 60\t head=source_path.read_text(encoding=\"utf-8\",errors=\"ignore\")[:4096]\n 61\t if \"TYPE=backup-matrix-generated\" in head or \"CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\" in head:\n 62\t continue\n 63\t destination_path.parent.mkdir(parents=True,exist_ok=True)\n 64\t temporary=destination_path.with_name(destination_path.name+\".tmp-ingest\")\n 65\t shutil.copy2(source_path,temporary)\n 66\t os.replace(temporary,destination_path)\n 67\t jid=str(job.get(\"id\",\"\"))\n 68\t name=str(job.get(\"name\",jid))\n 69\t timer=str(job.get(\"timer\",\"\"))\n 70\t service=str(job.get(\"service\",\"\"))\n 71\t enabled=cmd([\"systemctl\",\"is-enabled\",timer]) if timer else \"missing\"\n 72\t active=cmd([\"systemctl\",\"is-active\",timer]) if timer else \"missing\"\n 73\t source_kind=str(job.get(\"source_kind\",\"health_file\"))\n 74\t best=None\n 75\t for raw in job.get(\"evidence_inputs\",job.get(\"legacy_health_inputs\",[])):\n 76\t path=Path(raw)\n 77\t if not path.is_file():\n 78\t continue\n 79\t mtime=int(path.stat().st_mtime)\n 80\t if best is None or mtime>best[0]:\n 81\t best=(mtime,str(path),parse_health(path))\n 82\t service_result=cmd([\"systemctl\",\"show\",service,\"-p\",\"Result\",\"--value\"]) if service else \"missing\"\n 83\t service_exec_main_status=cmd([\"systemctl\",\"show\",service,\"-p\",\"ExecMainStatus\",\"--value\"]) if service else \"missing\"\n 84\t evidence_level=\"artifact_or_health\"\n 85\t if best is None and source_kind==\"systemd\":\n 86\t stamp=Path(\"/var/lib/systemd/timers\")/(\"stamp-\"+timer)\n 87\t mtime=int(stamp.stat().st_mtime) if stamp.is_file() else 0\n 88\t source=\"systemd:\"+service\n 89\t source_status=\"OK\" if service_result==\"success\" and service_exec_main_status==\"0\" else \"FAIL\"\n 90\t evidence_level=\"execution_only\"\n 91\t else:\n 92\t source=best[1] if best else \"\"\n 93\t source_status=normalize(best[2]) if best else \"UNMAPPED\"\n 94\t mtime=best[0] if best else 0\n 95\t age=now-mtime if mtime else -1\n 96\t max_age=int(job.get(\"max_age_seconds\",172800))\n 97\t critical=bool(job.get(\"critical\",False))\n 98\t if not critical and enabled!=\"enabled\":\n 99\t final=\"DISABLED\"\n 100\t elif critical and (enabled!=\"enabled\" or active!=\"active\"):\n 101\t final=\"FAIL\"\n 102\t elif source_status==\"UNMAPPED\":\n 103\t final=\"UNMAPPED\"\n 104\t elif source_status!=\"OK\":\n 105\t final=\"FAIL\"\n 106\t elif age<0:\n 107\t final=\"MISSING\"\n 108\t elif age>max_age:\n 109\t final=\"STALE\"\n 110\t else:\n 111\t final=\"OK\"\n 112\t freshness=\"UNKNOWN\" if age<0 else (\"FRESH\" if age<=max_age else \"STALE\")\n 113\t source_mtime=time.strftime(\"%Y-%m-%dT%H:%M:%SZ\",time.gmtime(mtime)) if mtime else \"\"\n 114\t result={\"id\":jid,\"name\":name,\"category\":job.get(\"category\",\"unknown\"),\"critical\":critical,\"timer\":timer,\"service\":service,\"timer_enabled\":enabled,\"timer_active\":active,\"status\":final,\"source_kind\":source_kind,\"evidence_level\":evidence_level,\"service_result\":service_result,\"service_exec_main_status\":service_exec_main_status,\"source_health\":source,\"source_status\":source_status,\"source_mtime\":source_mtime,\"age_seconds\":age,\"max_age_seconds\":max_age,\"freshness\":freshness,\"generated_health_output\":job.get(\"generated_health_output\",\"\")}\n 115\t results.append(result)\n 116\t mode_value=matrix.get(\"mode\",\"shadow\"); lines=[f\"STATUS={final}\",f\"TS={now_iso}\",\"TYPE=backup-matrix-generated\",f\"JOB_ID={jid}\",f\"JOB_NAME={name}\",f\"MATRIX_MODE={mode_value}\",f\"TIMER={timer}\",f\"SERVICE={service}\",f\"TIMER_ENABLED={enabled}\",f\"TIMER_ACTIVE={active}\",f\"SOURCE_KIND={source_kind}\",f\"EVIDENCE_LEVEL={evidence_level}\",f\"SERVICE_RESULT={service_result}\",f\"SERVICE_EXEC_MAIN_STATUS={service_exec_main_status}\",f\"SOURCE_HEALTH={source}\",f\"SOURCE_STATUS={source_status}\",f\"SOURCE_MTIME_UTC={source_mtime}\",f\"AGE_SECONDS={age}\",f\"MAX_AGE_SECONDS={max_age}\",f\"FRESHNESS={freshness}\"]\n 117\t atomic(Path(job.get(\"generated_health_output\",str(RENDER/(name+\".txt\")))),chr(10).join(lines)+chr(10))\n 118\t for raw in job.get(\"compatibility_outputs\",[]):\n 119\t legacy=str(raw)\n 120\t digest=hashlib.sha256(legacy.encode(\"utf-8\")).hexdigest()[:12]\n 121\t base_name=Path(legacy).name or \"health\"\n 122\t safe=f\"{jid}--{base_name}--{digest}.txt\"\n 123\t compat_path=COMPAT/safe\n 124\t compat_lines=lines+[f\"LEGACY_SOURCE_PATH={legacy}\",f\"CANONICAL_JOB_ID={jid}\",f\"CANONICAL_STATE={ROOT}/state.json\"]\n 125\t atomic(compat_path,chr(10).join(compat_lines)+chr(10))\n 126\t atomic(Path(legacy),chr(10).join(compat_lines)+chr(10))\n 127\t compat_map.append({\"job_id\":jid,\"legacy_path\":legacy,\"compatibility_path\":str(compat_path)})\n 128\tbad=sum(1 for item in results if item[\"status\"] in {\"FAIL\",\"STALE\",\"MISSING\"})\n 129\tunmapped=sum(1 for item in results if item[\"status\"]==\"UNMAPPED\")\n 130\tdisabled=sum(1 for item in results if item[\"status\"]==\"DISABLED\")\n 131\tok=sum(1 for item in results if item[\"status\"]==\"OK\")\n 132\tmode=str(matrix.get(\"mode\",\"shadow\"))\n 133\toverall=\"OK\" if bad==0 and unmapped==0 else (\"WARN\" if mode==\"shadow\" else \"FAIL\")\n 134\tcompat_map=sorted(compat_map,key=lambda item:(item[\"legacy_path\"],item[\"job_id\"]))\n 135\tatomic(ROOT/\"compat-map.json\",json.dumps(compat_map,ensure_ascii=False,indent=2,sort_keys=True)+chr(10))\n 136\tstate={\"version\":1,\"generated_at\":now_iso,\"matrix_mode\":mode,\"matrix_sha256\":hashlib.sha256(M.read_bytes()).hexdigest(),\"summary\":{\"status\":overall,\"jobs_total\":len(results),\"jobs_ok\":ok,\"jobs_bad\":bad,\"jobs_unmapped\":unmapped,\"jobs_disabled\":disabled},\"jobs\":results}\n 137\tstate[\"authority\"]=\"canonical\"\n 138\tstate[\"compatibility_map\"]=str(ROOT/\"compat-map.json\")\n 139\tstate[\"summary\"][\"compatibility_outputs\"]=len(compat_map)\n 140\tatomic(ROOT/\"state.json\",json.dumps(state,ensure_ascii=False,indent=2,sort_keys=True)+chr(10))\n===== FILE /usr/local/sbin/homelab-vm160-remote-backup =====\n/usr/local/sbin/homelab-vm160-remote-backup: Bourne-Again shell script, ASCII text executable\n 1\t#!/usr/bin/env bash\n 2\tset -Eeuo pipefail\n 3\texport SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK\n 4\tVMID=160\n 5\tSOURCE=root@[PRIVATE_IP]\n 6\tTARGET=root@[PRIVATE_IP]\n 7\tKH=[SENSITIVE_PATH]\n 8\tPARENT=/mnt/staging/vzdump\n 9\tPREFIX=vm160-pve02-\n 10\tHEALTH=/var/lib/homelab-health/vm160-remote-backup.txt\n 11\tLOCK=/run/lock/homelab-vm160-remote-backup.lock\n 12\tSSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=\"$KH\")\n 13\tTARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker\n 14\ttarget_worker(){ ssh \"${SSH[@]}\" \"$TARGET\" bash -s -- \"$@\" < \"$TARGET_WORKER\"; }\n 15\tinstall -d -m 0755 \"$(dirname \"$HEALTH\")\" \"$(dirname \"$LOCK\")\"\n 16\texec 9>\"$LOCK\"\n 17\tflock -n 9\n 18\ttest \"$(hostname -s)\" = pve01\n 19\ttest -x \"$TARGET_WORKER\"\n 20\ttest \"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"qm status $VMID\" | awk '{print $2}')\" = running\n 21\tSOURCE_ACTIVE_RC=0\n 22\tSOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n 23\ttest \"$SOURCE_ACTIVE_RC\" -eq 0 || test \"$SOURCE_ACTIVE_RC\" -eq 1\n 24\ttest -z \"$SOURCE_ACTIVE\"\n 25\tTS=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n 26\tDIR=\"$PARENT/${PREFIX}${TS}\"\n 27\tNAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n 28\tPART=\"$DIR/.${NAME}.partial\"\n 29\tFINAL=\"$DIR/$NAME\"\n 30\tLOG=\"$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)\"\n 31\tcleanup(){\n 32\t rc=$?\n 33\t trap - EXIT\n 34\t if test \"$rc\" -ne 0; then\n 35\t ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n 36\t tmp=\"${HEALTH}.tmp\"\n 37\t printf 'STATUS=FAIL\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nTARGET_DIR=%s\\nRC=%s\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$rc\" > \"$tmp\"\n 38\t chmod 0644 \"$tmp\"\n 39\t mv \"$tmp\" \"$HEALTH\"\n 40\t fi\n 41\t rm -f \"$LOG\"\n 42\t exit \"$rc\"\n 43\t}\n 44\ttrap cleanup EXIT\n 45\tREMOTE_FREE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"df -B1 --output=avail '$PARENT' | awk 'NR==2{print \\$1}'\")\"\n 46\tLATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n 47\tLATEST_SIZE=\"${LATEST_SIZE:-0}\"\n 48\tif test \"$LATEST_SIZE\" -gt 0; then\n 49\t REQUIRED=\"$((LATEST_SIZE + LATEST_SIZE / 4 + 10737418240))\"\n 50\telse\n 51\t REQUIRED=161061273600\n 52\tfi\n 53\ttest \"$REMOTE_FREE\" -ge \"$REQUIRED\"\n 54\tssh \"${SSH[@]}\" \"$TARGET\" \"install -d -m 0700 '$DIR'\"\n 55\tssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n 56\ttarget_worker finalize \"$PARENT\" \"$PREFIX\" \"$PART\" \"$FINAL\" \"$DIR\" \"$NAME\"\n 57\tssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n 58\tSIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"stat -c %s '$FINAL'\")\"\n 59\tSHA=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"sha256sum '$FINAL' | awk '{print \\$1}'\")\"\n 60\ttarget_worker promote \"$PARENT\" \"$PREFIX\" \"$DIR\" \"$TS\"\n 61\ttmp=\"${HEALTH}.tmp\"\n 62\tprintf 'STATUS=OK\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nSOURCE_NODE=pve02\\nTARGET_NODE=pve03\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\nREMOTE_RETENTION_GENERATIONS=1\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\" > \"$tmp\"\n 63\tchmod 0644 \"$tmp\"\n 64\tmv \"$tmp\" \"$HEALTH\"\n 65\ttrap - EXIT\n 66\trm -f \"$LOG\"\n 67\tprintf 'VM160_REMOTE_BACKUP_STATUS=OK\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\n' \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\"\n===== FILE /usr/local/sbin/homelab-cluster-admin-pull =====\n/usr/local/sbin/homelab-cluster-admin-pull: Bourne-Again shell script, ASCII text executable, with very long lines (424)\n 1\t#!/usr/bin/env bash\n 2\tset +e +u +o pipefail 2>/dev/null || true\n 3\tTS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\n 4\tROOT=/var/lib/homelab-cluster-admin\n 5\tHEALTH_ROOT=/var/lib/homelab-health\n 6\tmkdir -p \"$ROOT\" \"$HEALTH_ROOT\"\n 7\tREMOTE=$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf \"__OBS__\\\\n\"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf \"__FULL__\\\\n\"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err)\n 8\tRC=$?\n 9\tOBS=$(printf \"%s\\n\" \"$REMOTE\" | awk '/^__OBS__$/{f=1;next}/^__FULL__$/{f=0}f')\n 10\tFULL=$(printf \"%s\\n\" \"$REMOTE\" | awk '/^__FULL__$/{f=1;next}f')\n 11\tOBS_BAD=0; FULL_BAD=0\n 12\tprintf \"%s\\n\" \"$OBS\" | grep -q \"^STATUS=OK\" || OBS_BAD=1\n 13\tprintf \"%s\\n\" \"$FULL\" | grep -q \"^STATUS=OK\" || FULL_BAD=1\n 14\tprintf \"%s\\n\" \"$FULL\" | grep -q \"BAD=0\" || FULL_BAD=1\n 15\tPROBE_OK=$(printf \"%s\\n\" \"$FULL\" | sed -n 's/.*PROBE_OK=\\([0-9][0-9]*\\).*/\\1/p' | head -n1)\n 16\tPROBE_BAD=$(printf \"%s\\n\" \"$FULL\" | sed -n 's/.*PROBE_BAD=\\([0-9][0-9]*\\).*/\\1/p' | head -n1)\n 17\t[ \"${PROBE_BAD:-1}\" = 0 ] || FULL_BAD=1\n 18\tBAD=$((OBS_BAD+FULL_BAD)); [ \"$RC\" -eq 0 ] || BAD=$((BAD+1))\n 19\tprintf \"%s\\n\" \"$OBS\" > \"$ROOT/observer-remote-latest.txt\"\n 20\tprintf \"%s\\n\" \"$FULL\" > \"$ROOT/full-observer-central-latest.md\"\n 21\tOBS_STATUS=$([ \"$OBS_BAD\" -eq 0 ] && [ \"$RC\" -eq 0 ] && echo OK || echo WARN)\n 22\tFULL_STATUS=$([ \"$FULL_BAD\" -eq 0 ] && [ \"$RC\" -eq 0 ] && echo OK || echo WARN)\n 23\techo \"STATUS=$OBS_STATUS TS=$TS TYPE=cluster-admin-central-pull BAD=$OBS_BAD VMID=180 IP=[PRIVATE_IP] SSH_RC=$RC OBSERVER_STATUS=$OBS_STATUS REPORT=$ROOT/observer-remote-latest.txt\" > \"$HEALTH_ROOT/cluster-admin-observer.txt\"\n 24\techo \"STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-full-observer-central BAD=$FULL_BAD WARN=0 VMID=180 PULL_RC=$RC FULL_STATUS=$FULL_STATUS PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} REPORT=$ROOT/full-observer-central-latest.md\" > \"$HEALTH_ROOT/cluster-admin-full-observer.txt\"\n 25\techo \"STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-restricted-probes BAD=$FULL_BAD VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\" > \"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\"\n 26\tcat \"$HEALTH_ROOT/cluster-admin-observer.txt\"\n 27\tcat \"$HEALTH_ROOT/cluster-admin-full-observer.txt\"\n 28\tcat \"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\"\n 29\t[ \"$BAD\" -eq 0 ]\nCHANGES_MADE=NO\n" + "output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217\n=== LEDGER_217 ===\n2026-07-22T15:35:58Z\tBACKUP-FAILED-UNITS-SOURCE-217\t2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86\t0\tOK\tcluster-audit\n=== MATRIX_ENTRYPOINT ===\n 1\t#!/usr/bin/env python3\n 2\tfrom pathlib import Path\n 3\timport shutil\n 4\timport json,subprocess,os,time,sys,hashlib\n 5\tM=Path(\"/etc/homelab-backup-matrix/matrix.json\")\n 6\tROOT=Path(\"/var/lib/homelab-backup-matrix\")\n 7\tRENDER=ROOT/\"rendered\"\n 8\tCOMPAT=ROOT/\"compat\"\n 9\tdef cmd(args):\n 10\t p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)\n 11\t return p.stdout.strip() or \"unknown\"\n 12\tdef atomic(path,text,mode=0o640):\n 13\t path.parent.mkdir(parents=True,exist_ok=True)\n 14\t tmp=path.with_name(path.name+\".tmp\")\n 15\t tmp.write_text(text,encoding=\"utf-8\")\n 16\t os.chmod(tmp,mode)\n 17\t os.replace(tmp,path)\n 18\tdef parse_health(path):\n=== VM160_CONTROL_FLOW ===\n5:SOURCE=root@[PRIVATE_IP]\n6:TARGET=root@[PRIVATE_IP]\n7:KH=[SENSITIVE_PATH]\n8:PARENT=/mnt/staging/vzdump\n11:LOCK=/run/lock/homelab-vm160-remote-backup.lock\n13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker\n14:target_worker(){ ssh \"${SSH[@]}\" \"$TARGET\" bash -s -- \"$@\" < \"$TARGET_WORKER\"; }\n15:install -d -m 0755 \"$(dirname \"$HEALTH\")\" \"$(dirname \"$LOCK\")\"\n16:exec 9>\"$LOCK\"\n19:test -x \"$TARGET_WORKER\"\n20:test \"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"qm status $VMID\" | awk '{print $2}')\" = running\n21:SOURCE_ACTIVE_RC=0\n22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n23:test \"$SOURCE_ACTIVE_RC\" -eq 0 || test \"$SOURCE_ACTIVE_RC\" -eq 1\n24:test -z \"$SOURCE_ACTIVE\"\n27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n35: ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n37: printf 'STATUS=FAIL\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nTARGET_DIR=%s\\nRC=%s\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$rc\" > \"$tmp\"\n44:trap cleanup EXIT\n45:REMOTE_FREE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"df -B1 --output=avail '$PARENT' | awk 'NR==2{print \\$1}'\")\"\n46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n48:if test \"$LATEST_SIZE\" -gt 0; then\n50:else\n52:fi\n53:test \"$REMOTE_FREE\" -ge \"$REQUIRED\"\n54:ssh \"${SSH[@]}\" \"$TARGET\" \"install -d -m 0700 '$DIR'\"\n55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n58:SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"stat -c %s '$FINAL'\")\"\n59:SHA=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"sha256sum '$FINAL' | awk '{print \\$1}'\")\"\n62:printf 'STATUS=OK\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nSOURCE_NODE=pve02\\nTARGET_NODE=pve03\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\nREMOTE_RETENTION_GENERATIONS=1\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\" > \"$tmp\"\n65:trap - EXIT\n67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\n' \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\"\n=== CLUSTER_ADMIN_CURRENT_FAILURE ===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index a62ff62..69b65a0 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217 +COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218 STATUS=OK RC=0 HOST=pve01 @@ -9,48 +9,20 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86 +COMMAND_SHA256=67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=96e72a8cec4b5e5cde52a48e6816f4ce162311b07530ee398c57efc67f3480b1 -SANITIZED_OUTPUT_SHA256=dbe6bb8b0419312d9774755ff8668c435118bc102b49b0255667d04f06dda499 +RAW_EVIDENCE_SHA256=ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1 +SANITIZED_OUTPUT_SHA256=d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733 OUTPUT_BEGIN -PREVIOUS_COMMAND_ID=RUNTIME-PUBLISH-GAP-DIAG-216 -===== UNIT homelab-backup-matrix-refresh ===== -# /etc/systemd/system/homelab-backup-matrix-refresh.service -[Unit] -Description=Generate unified homelab backup matrix state -After=local-fs.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-backup-matrix-refresh -TimeoutStartSec=5min -===== UNIT homelab-vm160-remote-backup ===== -# /etc/systemd/system/homelab-vm160-remote-backup.service -[Unit] -Description=VM160 full-image backup streamed from pve02 to pve03 -After=network-online.target pve-cluster.service -Wants=network-online.target - -[Service] -Type=oneshot -TimeoutStartSec=6h -ExecStart=/usr/local/sbin/homelab-vm160-remote-backup -===== UNIT homelab-cluster-admin-pull ===== -# /etc/systemd/system/homelab-cluster-admin-pull.service -[Unit] -Description=Pull cluster-admin observer health into pve01 - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-cluster-admin-pull -===== FILE /usr/local/sbin/homelab-backup-matrix-refresh ===== -/usr/local/sbin/homelab-backup-matrix-refresh: Python script, ASCII text executable, with very long lines (592) +PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217 +=== LEDGER_217 === +2026-07-22T15:35:58Z BACKUP-FAILED-UNITS-SOURCE-217 2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86 0 OK cluster-audit +=== MATRIX_ENTRYPOINT === 1 #!/usr/bin/env python3 2 from pathlib import Path 3 import shutil @@ -69,228 +41,43 @@ ExecStart=/usr/local/sbin/homelab-cluster-admin-pull 16 os.chmod(tmp,mode) 17 os.replace(tmp,path) 18 def parse_health(path): - 19 if path.suffix.lower()==".json": - 20 try: - 21 data=json.loads(path.read_text(encoding="utf-8",errors="replace")) - 22 return str(data.get("status",data.get("STATUS",data.get("result","UNKNOWN")))) - 23 except Exception: - 24 return "ERROR" - 25 data={} - 26 text=path.read_text(encoding="utf-8",errors="replace") - 27 for token in text.replace(chr(10)," ").split(): - 28 if "=" in token: - 29 key,value=token.split("=",1) - 30 if key and key[0].isalpha(): - 31 data[key]=value - 32 return str(data.get("STATUS",data.get("status","UNKNOWN"))) - 33 def normalize(value): - 34 value=str(value).strip().upper() - 35 if value in {"OK","PASS","PASSED","SUCCESS","HEALTHY"}: - 36 return "OK" - 37 if value in {"WARN","WARNING"}: - 38 return "WARN" - 39 if value in {"FAIL","FAILED","ERROR","CRITICAL","BAD"}: - 40 return "FAIL" - 41 return value or "UNKNOWN" - 42 matrix=json.loads(M.read_text(encoding="utf-8")) - 43 ROOT.mkdir(parents=True,exist_ok=True) - 44 RENDER.mkdir(parents=True,exist_ok=True) - 45 COMPAT.mkdir(parents=True,exist_ok=True) - 46 now=int(time.time()) - 47 now_iso=time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime(now)) - 48 results=[] - 49 compat_map=[] - 50 for job in matrix.get("jobs",[]): - 51 original_sources=list(job.get("evidence_sources_original",[])) - 52 evidence_inputs=list(job.get("evidence_inputs",[])) - 53 if original_sources and len(original_sources)!=len(evidence_inputs): - 54 raise RuntimeError("evidence_mapping_length_mismatch:"+str(job.get("name",""))) - 55 for source_raw,destination_raw in zip(original_sources,evidence_inputs): - 56 source_path=Path(source_raw) - 57 destination_path=Path(destination_raw) - 58 if not source_path.is_file(): - 59 continue - 60 head=source_path.read_text(encoding="utf-8",errors="ignore")[:4096] - 61 if "TYPE=backup-matrix-generated" in head or "CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json" in head: - 62 continue - 63 destination_path.parent.mkdir(parents=True,exist_ok=True) - 64 temporary=destination_path.with_name(destination_path.name+".tmp-ingest") - 65 shutil.copy2(source_path,temporary) - 66 os.replace(temporary,destination_path) - 67 jid=str(job.get("id","")) - 68 name=str(job.get("name",jid)) - 69 timer=str(job.get("timer","")) - 70 service=str(job.get("service","")) - 71 enabled=cmd(["systemctl","is-enabled",timer]) if timer else "missing" - 72 active=cmd(["systemctl","is-active",timer]) if timer else "missing" - 73 source_kind=str(job.get("source_kind","health_file")) - 74 best=None - 75 for raw in job.get("evidence_inputs",job.get("legacy_health_inputs",[])): - 76 path=Path(raw) - 77 if not path.is_file(): - 78 continue - 79 mtime=int(path.stat().st_mtime) - 80 if best is None or mtime>best[0]: - 81 best=(mtime,str(path),parse_health(path)) - 82 service_result=cmd(["systemctl","show",service,"-p","Result","--value"]) if service else "missing" - 83 service_exec_main_status=cmd(["systemctl","show",service,"-p","ExecMainStatus","--value"]) if service else "missing" - 84 evidence_level="artifact_or_health" - 85 if best is None and source_kind=="systemd": - 86 stamp=Path("/var/lib/systemd/timers")/("stamp-"+timer) - 87 mtime=int(stamp.stat().st_mtime) if stamp.is_file() else 0 - 88 source="systemd:"+service - 89 source_status="OK" if service_result=="success" and service_exec_main_status=="0" else "FAIL" - 90 evidence_level="execution_only" - 91 else: - 92 source=best[1] if best else "" - 93 source_status=normalize(best[2]) if best else "UNMAPPED" - 94 mtime=best[0] if best else 0 - 95 age=now-mtime if mtime else -1 - 96 max_age=int(job.get("max_age_seconds",172800)) - 97 critical=bool(job.get("critical",False)) - 98 if not critical and enabled!="enabled": - 99 final="DISABLED" - 100 elif critical and (enabled!="enabled" or active!="active"): - 101 final="FAIL" - 102 elif source_status=="UNMAPPED": - 103 final="UNMAPPED" - 104 elif source_status!="OK": - 105 final="FAIL" - 106 elif age<0: - 107 final="MISSING" - 108 elif age>max_age: - 109 final="STALE" - 110 else: - 111 final="OK" - 112 freshness="UNKNOWN" if age<0 else ("FRESH" if age<=max_age else "STALE") - 113 source_mtime=time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime(mtime)) if mtime else "" - 114 result={"id":jid,"name":name,"category":job.get("category","unknown"),"critical":critical,"timer":timer,"service":service,"timer_enabled":enabled,"timer_active":active,"status":final,"source_kind":source_kind,"evidence_level":evidence_level,"service_result":service_result,"service_exec_main_status":service_exec_main_status,"source_health":source,"source_status":source_status,"source_mtime":source_mtime,"age_seconds":age,"max_age_seconds":max_age,"freshness":freshness,"generated_health_output":job.get("generated_health_output","")} - 115 results.append(result) - 116 mode_value=matrix.get("mode","shadow"); lines=[f"STATUS={final}",f"TS={now_iso}","TYPE=backup-matrix-generated",f"JOB_ID={jid}",f"JOB_NAME={name}",f"MATRIX_MODE={mode_value}",f"TIMER={timer}",f"SERVICE={service}",f"TIMER_ENABLED={enabled}",f"TIMER_ACTIVE={active}",f"SOURCE_KIND={source_kind}",f"EVIDENCE_LEVEL={evidence_level}",f"SERVICE_RESULT={service_result}",f"SERVICE_EXEC_MAIN_STATUS={service_exec_main_status}",f"SOURCE_HEALTH={source}",f"SOURCE_STATUS={source_status}",f"SOURCE_MTIME_UTC={source_mtime}",f"AGE_SECONDS={age}",f"MAX_AGE_SECONDS={max_age}",f"FRESHNESS={freshness}"] - 117 atomic(Path(job.get("generated_health_output",str(RENDER/(name+".txt")))),chr(10).join(lines)+chr(10)) - 118 for raw in job.get("compatibility_outputs",[]): - 119 legacy=str(raw) - 120 digest=hashlib.sha256(legacy.encode("utf-8")).hexdigest()[:12] - 121 base_name=Path(legacy).name or "health" - 122 safe=f"{jid}--{base_name}--{digest}.txt" - 123 compat_path=COMPAT/safe - 124 compat_lines=lines+[f"LEGACY_SOURCE_PATH={legacy}",f"CANONICAL_JOB_ID={jid}",f"CANONICAL_STATE={ROOT}/state.json"] - 125 atomic(compat_path,chr(10).join(compat_lines)+chr(10)) - 126 atomic(Path(legacy),chr(10).join(compat_lines)+chr(10)) - 127 compat_map.append({"job_id":jid,"legacy_path":legacy,"compatibility_path":str(compat_path)}) - 128 bad=sum(1 for item in results if item["status"] in {"FAIL","STALE","MISSING"}) - 129 unmapped=sum(1 for item in results if item["status"]=="UNMAPPED") - 130 disabled=sum(1 for item in results if item["status"]=="DISABLED") - 131 ok=sum(1 for item in results if item["status"]=="OK") - 132 mode=str(matrix.get("mode","shadow")) - 133 overall="OK" if bad==0 and unmapped==0 else ("WARN" if mode=="shadow" else "FAIL") - 134 compat_map=sorted(compat_map,key=lambda item:(item["legacy_path"],item["job_id"])) - 135 atomic(ROOT/"compat-map.json",json.dumps(compat_map,ensure_ascii=False,indent=2,sort_keys=True)+chr(10)) - 136 state={"version":1,"generated_at":now_iso,"matrix_mode":mode,"matrix_sha256":hashlib.sha256(M.read_bytes()).hexdigest(),"summary":{"status":overall,"jobs_total":len(results),"jobs_ok":ok,"jobs_bad":bad,"jobs_unmapped":unmapped,"jobs_disabled":disabled},"jobs":results} - 137 state["authority"]="canonical" - 138 state["compatibility_map"]=str(ROOT/"compat-map.json") - 139 state["summary"]["compatibility_outputs"]=len(compat_map) - 140 atomic(ROOT/"state.json",json.dumps(state,ensure_ascii=False,indent=2,sort_keys=True)+chr(10)) -===== FILE /usr/local/sbin/homelab-vm160-remote-backup ===== -/usr/local/sbin/homelab-vm160-remote-backup: Bourne-Again shell script, ASCII text executable - 1 #!/usr/bin/env bash - 2 set -Eeuo pipefail - 3 export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK - 4 VMID=160 - 5 SOURCE=root@[PRIVATE_IP] - 6 TARGET=root@[PRIVATE_IP] - 7 KH=[SENSITIVE_PATH] - 8 PARENT=/mnt/staging/vzdump - 9 PREFIX=vm160-pve02- - 10 HEALTH=/var/lib/homelab-health/vm160-remote-backup.txt - 11 LOCK=/run/lock/homelab-vm160-remote-backup.lock - 12 SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$KH") - 13 TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker - 14 target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; } - 15 install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")" - 16 exec 9>"$LOCK" - 17 flock -n 9 - 18 test "$(hostname -s)" = pve01 - 19 test -x "$TARGET_WORKER" - 20 test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running - 21 SOURCE_ACTIVE_RC=0 - 22 SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$? - 23 test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1 - 24 test -z "$SOURCE_ACTIVE" - 25 TS="$(date -u +%Y%m%dT%H%M%SZ)" - 26 DIR="$PARENT/${PREFIX}${TS}" - 27 NAME="vzdump-qemu-${VMID}-${TS}.vma.zst" - 28 PART="$DIR/.${NAME}.partial" - 29 FINAL="$DIR/$NAME" - 30 LOG="$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)" - 31 cleanup(){ - 32 rc=$? - 33 trap - EXIT - 34 if test "$rc" -ne 0; then - 35 ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true - 36 tmp="${HEALTH}.tmp" - 37 printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp" - 38 chmod 0644 "$tmp" - 39 mv "$tmp" "$HEALTH" - 40 fi - 41 rm -f "$LOG" - 42 exit "$rc" - 43 } - 44 trap cleanup EXIT - 45 REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")" - 46 LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")" - 47 LATEST_SIZE="${LATEST_SIZE:-0}" - 48 if test "$LATEST_SIZE" -gt 0; then - 49 REQUIRED="$((LATEST_SIZE + LATEST_SIZE / 4 + 10737418240))" - 50 else - 51 REQUIRED=161061273600 - 52 fi - 53 test "$REMOTE_FREE" -ge "$REQUIRED" - 54 ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'" - 55 ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" - 56 target_worker finalize "$PARENT" "$PREFIX" "$PART" "$FINAL" "$DIR" "$NAME" - 57 ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" - 58 SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")" - 59 SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")" - 60 target_worker promote "$PARENT" "$PREFIX" "$DIR" "$TS" - 61 tmp="${HEALTH}.tmp" - 62 printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp" - 63 chmod 0644 "$tmp" - 64 mv "$tmp" "$HEALTH" - 65 trap - EXIT - 66 rm -f "$LOG" - 67 printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA" -===== FILE /usr/local/sbin/homelab-cluster-admin-pull ===== -/usr/local/sbin/homelab-cluster-admin-pull: Bourne-Again shell script, ASCII text executable, with very long lines (424) - 1 #!/usr/bin/env bash - 2 set +e +u +o pipefail 2>/dev/null || true - 3 TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) - 4 ROOT=/var/lib/homelab-cluster-admin - 5 HEALTH_ROOT=/var/lib/homelab-health - 6 mkdir -p "$ROOT" "$HEALTH_ROOT" - 7 REMOTE=$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf "__OBS__\\n"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf "__FULL__\\n"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err) - 8 RC=$? - 9 OBS=$(printf "%s\n" "$REMOTE" | awk '/^__OBS__$/{f=1;next}/^__FULL__$/{f=0}f') - 10 FULL=$(printf "%s\n" "$REMOTE" | awk '/^__FULL__$/{f=1;next}f') - 11 OBS_BAD=0; FULL_BAD=0 - 12 printf "%s\n" "$OBS" | grep -q "^STATUS=OK" || OBS_BAD=1 - 13 printf "%s\n" "$FULL" | grep -q "^STATUS=OK" || FULL_BAD=1 - 14 printf "%s\n" "$FULL" | grep -q "BAD=0" || FULL_BAD=1 - 15 PROBE_OK=$(printf "%s\n" "$FULL" | sed -n 's/.*PROBE_OK=\([0-9][0-9]*\).*/\1/p' | head -n1) - 16 PROBE_BAD=$(printf "%s\n" "$FULL" | sed -n 's/.*PROBE_BAD=\([0-9][0-9]*\).*/\1/p' | head -n1) - 17 [ "${PROBE_BAD:-1}" = 0 ] || FULL_BAD=1 - 18 BAD=$((OBS_BAD+FULL_BAD)); [ "$RC" -eq 0 ] || BAD=$((BAD+1)) - 19 printf "%s\n" "$OBS" > "$ROOT/observer-remote-latest.txt" - 20 printf "%s\n" "$FULL" > "$ROOT/full-observer-central-latest.md" - 21 OBS_STATUS=$([ "$OBS_BAD" -eq 0 ] && [ "$RC" -eq 0 ] && echo OK || echo WARN) - 22 FULL_STATUS=$([ "$FULL_BAD" -eq 0 ] && [ "$RC" -eq 0 ] && echo OK || echo WARN) - 23 echo "STATUS=$OBS_STATUS TS=$TS TYPE=cluster-admin-central-pull BAD=$OBS_BAD VMID=180 IP=[PRIVATE_IP] SSH_RC=$RC OBSERVER_STATUS=$OBS_STATUS REPORT=$ROOT/observer-remote-latest.txt" > "$HEALTH_ROOT/cluster-admin-observer.txt" - 24 echo "STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-full-observer-central BAD=$FULL_BAD WARN=0 VMID=180 PULL_RC=$RC FULL_STATUS=$FULL_STATUS PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} REPORT=$ROOT/full-observer-central-latest.md" > "$HEALTH_ROOT/cluster-admin-full-observer.txt" - 25 echo "STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-restricted-probes BAD=$FULL_BAD VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer" > "$HEALTH_ROOT/cluster-admin-restricted-probes.txt" - 26 cat "$HEALTH_ROOT/cluster-admin-observer.txt" - 27 cat "$HEALTH_ROOT/cluster-admin-full-observer.txt" - 28 cat "$HEALTH_ROOT/cluster-admin-restricted-probes.txt" - 29 [ "$BAD" -eq 0 ] +=== VM160_CONTROL_FLOW === +5:SOURCE=root@[PRIVATE_IP] +6:TARGET=root@[PRIVATE_IP] +7:KH=[SENSITIVE_PATH] +8:PARENT=/mnt/staging/vzdump +11:LOCK=/run/lock/homelab-vm160-remote-backup.lock +13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker +14:target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; } +15:install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")" +16:exec 9>"$LOCK" +19:test -x "$TARGET_WORKER" +20:test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running +21:SOURCE_ACTIVE_RC=0 +22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$? +23:test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1 +24:test -z "$SOURCE_ACTIVE" +27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst" +35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true +37: printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp" +44:trap cleanup EXIT +45:REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")" +46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")" +48:if test "$LATEST_SIZE" -gt 0; then +50:else +52:fi +53:test "$REMOTE_FREE" -ge "$REQUIRED" +54:ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'" +55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" +57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" +58:SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")" +59:SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")" +62:printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp" +65:trap - EXIT +67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA" +=== CLUSTER_ADMIN_CURRENT_FAILURE === +/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md +/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md CHANGES_MADE=NO OUTPUT_END