runtime: publish BACKUP-FAILED-UNITS-DECISION-218
This commit is contained in:
homelab-runtime-publisher
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"channel": "homelab-runtime",
|
||||
"command_id": "BACKUP-FAILED-UNITS-DECISION-218",
|
||||
"status": "OK",
|
||||
"rc": 0,
|
||||
"host": "pve01",
|
||||
"mode": "read-only",
|
||||
"component": "cluster-audit",
|
||||
"started_at_utc": "2026-07-22T15:39:09Z",
|
||||
"finished_at_utc": "2026-07-22T15:39:09Z",
|
||||
"reference_register_checked": true,
|
||||
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
|
||||
"error_register_checked": true,
|
||||
"error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
|
||||
"command_sha256": "67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730",
|
||||
"duplicate_failed_command_blocked": false,
|
||||
"block_reason": null,
|
||||
"execution_started": true,
|
||||
"changes_made": false,
|
||||
"sanitized": true,
|
||||
"secrets_included": false,
|
||||
"private_addresses_included": false,
|
||||
"raw_evidence_retained_locally": true,
|
||||
"raw_evidence_sha256": "ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1",
|
||||
"sanitized_output_sha256": "d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733",
|
||||
"output_truncated_in_json": false,
|
||||
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
||||
"output": "PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217\n=== LEDGER_217 ===\n2026-07-22T15:35:58Z\tBACKUP-FAILED-UNITS-SOURCE-217\t2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86\t0\tOK\tcluster-audit\n=== MATRIX_ENTRYPOINT ===\n 1\t#!/usr/bin/env python3\n 2\tfrom pathlib import Path\n 3\timport shutil\n 4\timport json,subprocess,os,time,sys,hashlib\n 5\tM=Path(\"/etc/homelab-backup-matrix/matrix.json\")\n 6\tROOT=Path(\"/var/lib/homelab-backup-matrix\")\n 7\tRENDER=ROOT/\"rendered\"\n 8\tCOMPAT=ROOT/\"compat\"\n 9\tdef cmd(args):\n 10\t p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)\n 11\t return p.stdout.strip() or \"unknown\"\n 12\tdef atomic(path,text,mode=0o640):\n 13\t path.parent.mkdir(parents=True,exist_ok=True)\n 14\t tmp=path.with_name(path.name+\".tmp\")\n 15\t tmp.write_text(text,encoding=\"utf-8\")\n 16\t os.chmod(tmp,mode)\n 17\t os.replace(tmp,path)\n 18\tdef parse_health(path):\n=== VM160_CONTROL_FLOW ===\n5:SOURCE=root@[PRIVATE_IP]\n6:TARGET=root@[PRIVATE_IP]\n7:KH=[SENSITIVE_PATH]\n8:PARENT=/mnt/staging/vzdump\n11:LOCK=/run/lock/homelab-vm160-remote-backup.lock\n13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker\n14:target_worker(){ ssh \"${SSH[@]}\" \"$TARGET\" bash -s -- \"$@\" < \"$TARGET_WORKER\"; }\n15:install -d -m 0755 \"$(dirname \"$HEALTH\")\" \"$(dirname \"$LOCK\")\"\n16:exec 9>\"$LOCK\"\n19:test -x \"$TARGET_WORKER\"\n20:test \"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"qm status $VMID\" | awk '{print $2}')\" = running\n21:SOURCE_ACTIVE_RC=0\n22:SOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n23:test \"$SOURCE_ACTIVE_RC\" -eq 0 || test \"$SOURCE_ACTIVE_RC\" -eq 1\n24:test -z \"$SOURCE_ACTIVE\"\n27:NAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n35: ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n37: printf 'STATUS=FAIL\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nTARGET_DIR=%s\\nRC=%s\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$rc\" > \"$tmp\"\n44:trap cleanup EXIT\n45:REMOTE_FREE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"df -B1 --output=avail '$PARENT' | awk 'NR==2{print \\$1}'\")\"\n46:LATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n48:if test \"$LATEST_SIZE\" -gt 0; then\n50:else\n52:fi\n53:test \"$REMOTE_FREE\" -ge \"$REQUIRED\"\n54:ssh \"${SSH[@]}\" \"$TARGET\" \"install -d -m 0700 '$DIR'\"\n55:ssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n57:ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n58:SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"stat -c %s '$FINAL'\")\"\n59:SHA=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"sha256sum '$FINAL' | awk '{print \\$1}'\")\"\n62:printf 'STATUS=OK\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nSOURCE_NODE=pve02\\nTARGET_NODE=pve03\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\nREMOTE_RETENTION_GENERATIONS=1\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\" > \"$tmp\"\n65:trap - EXIT\n67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\n' \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\"\n=== CLUSTER_ADMIN_CURRENT_FAILURE ===\n/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\n/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\nCHANGES_MADE=NO\n"
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
MODE=read-only
|
||||
COMPONENT=cluster-audit
|
||||
REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||
COMMAND_SHA256=67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGES_MADE=false
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1
|
||||
SANITIZED_OUTPUT_SHA256=d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733
|
||||
OUTPUT_BEGIN
|
||||
PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217
|
||||
=== LEDGER_217 ===
|
||||
2026-07-22T15:35:58Z BACKUP-FAILED-UNITS-SOURCE-217 2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86 0 OK cluster-audit
|
||||
=== MATRIX_ENTRYPOINT ===
|
||||
1 #!/usr/bin/env python3
|
||||
2 from pathlib import Path
|
||||
3 import shutil
|
||||
4 import json,subprocess,os,time,sys,hashlib
|
||||
5 M=Path("/etc/homelab-backup-matrix/matrix.json")
|
||||
6 ROOT=Path("/var/lib/homelab-backup-matrix")
|
||||
7 RENDER=ROOT/"rendered"
|
||||
8 COMPAT=ROOT/"compat"
|
||||
9 def cmd(args):
|
||||
10 p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)
|
||||
11 return p.stdout.strip() or "unknown"
|
||||
12 def atomic(path,text,mode=0o640):
|
||||
13 path.parent.mkdir(parents=True,exist_ok=True)
|
||||
14 tmp=path.with_name(path.name+".tmp")
|
||||
15 tmp.write_text(text,encoding="utf-8")
|
||||
16 os.chmod(tmp,mode)
|
||||
17 os.replace(tmp,path)
|
||||
18 def parse_health(path):
|
||||
=== VM160_CONTROL_FLOW ===
|
||||
5:SOURCE=root@[PRIVATE_IP]
|
||||
6:TARGET=root@[PRIVATE_IP]
|
||||
7:KH=[SENSITIVE_PATH]
|
||||
8:PARENT=/mnt/staging/vzdump
|
||||
11:LOCK=/run/lock/homelab-vm160-remote-backup.lock
|
||||
13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker
|
||||
14:target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; }
|
||||
15:install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")"
|
||||
16:exec 9>"$LOCK"
|
||||
19:test -x "$TARGET_WORKER"
|
||||
20:test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running
|
||||
21:SOURCE_ACTIVE_RC=0
|
||||
22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$?
|
||||
23:test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1
|
||||
24:test -z "$SOURCE_ACTIVE"
|
||||
27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst"
|
||||
35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true
|
||||
37: printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp"
|
||||
44:trap cleanup EXIT
|
||||
45:REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")"
|
||||
46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")"
|
||||
48:if test "$LATEST_SIZE" -gt 0; then
|
||||
50:else
|
||||
52:fi
|
||||
53:test "$REMOTE_FREE" -ge "$REQUIRED"
|
||||
54:ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'"
|
||||
55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'"
|
||||
57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG"
|
||||
58:SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")"
|
||||
59:SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")"
|
||||
62:printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp"
|
||||
65:trap - EXIT
|
||||
67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA"
|
||||
=== CLUSTER_ADMIN_CURRENT_FAILURE ===
|
||||
/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
|
||||
/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
|
||||
CHANGES_MADE=NO
|
||||
|
||||
OUTPUT_END
|
||||
CHAT_OUTPUT_END
|
||||
+7
-7
File diff suppressed because one or more lines are too long
+45
-258
@@ -1,5 +1,5 @@
|
||||
CHAT_OUTPUT_BEGIN
|
||||
COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217
|
||||
COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218
|
||||
STATUS=OK
|
||||
RC=0
|
||||
HOST=pve01
|
||||
@@ -9,48 +9,20 @@ REFERENCE_REGISTER_CHECK=OK
|
||||
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
||||
ERROR_REGISTER_CHECK=OK
|
||||
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
||||
COMMAND_SHA256=2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86
|
||||
COMMAND_SHA256=67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730
|
||||
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
||||
EXECUTION_STARTED=true
|
||||
CHANGES_MADE=false
|
||||
SANITIZED=yes
|
||||
SECRETS_INCLUDED=no
|
||||
PRIVATE_ADDRESSES_INCLUDED=no
|
||||
RAW_EVIDENCE_SHA256=96e72a8cec4b5e5cde52a48e6816f4ce162311b07530ee398c57efc67f3480b1
|
||||
SANITIZED_OUTPUT_SHA256=dbe6bb8b0419312d9774755ff8668c435118bc102b49b0255667d04f06dda499
|
||||
RAW_EVIDENCE_SHA256=ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1
|
||||
SANITIZED_OUTPUT_SHA256=d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733
|
||||
OUTPUT_BEGIN
|
||||
PREVIOUS_COMMAND_ID=RUNTIME-PUBLISH-GAP-DIAG-216
|
||||
===== UNIT homelab-backup-matrix-refresh =====
|
||||
# /etc/systemd/system/homelab-backup-matrix-refresh.service
|
||||
[Unit]
|
||||
Description=Generate unified homelab backup matrix state
|
||||
After=local-fs.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/homelab-backup-matrix-refresh
|
||||
TimeoutStartSec=5min
|
||||
===== UNIT homelab-vm160-remote-backup =====
|
||||
# /etc/systemd/system/homelab-vm160-remote-backup.service
|
||||
[Unit]
|
||||
Description=VM160 full-image backup streamed from pve02 to pve03
|
||||
After=network-online.target pve-cluster.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
TimeoutStartSec=6h
|
||||
ExecStart=/usr/local/sbin/homelab-vm160-remote-backup
|
||||
===== UNIT homelab-cluster-admin-pull =====
|
||||
# /etc/systemd/system/homelab-cluster-admin-pull.service
|
||||
[Unit]
|
||||
Description=Pull cluster-admin observer health into pve01
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/homelab-cluster-admin-pull
|
||||
===== FILE /usr/local/sbin/homelab-backup-matrix-refresh =====
|
||||
/usr/local/sbin/homelab-backup-matrix-refresh: Python script, ASCII text executable, with very long lines (592)
|
||||
PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217
|
||||
=== LEDGER_217 ===
|
||||
2026-07-22T15:35:58Z BACKUP-FAILED-UNITS-SOURCE-217 2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86 0 OK cluster-audit
|
||||
=== MATRIX_ENTRYPOINT ===
|
||||
1 #!/usr/bin/env python3
|
||||
2 from pathlib import Path
|
||||
3 import shutil
|
||||
@@ -69,228 +41,43 @@ ExecStart=/usr/local/sbin/homelab-cluster-admin-pull
|
||||
16 os.chmod(tmp,mode)
|
||||
17 os.replace(tmp,path)
|
||||
18 def parse_health(path):
|
||||
19 if path.suffix.lower()==".json":
|
||||
20 try:
|
||||
21 data=json.loads(path.read_text(encoding="utf-8",errors="replace"))
|
||||
22 return str(data.get("status",data.get("STATUS",data.get("result","UNKNOWN"))))
|
||||
23 except Exception:
|
||||
24 return "ERROR"
|
||||
25 data={}
|
||||
26 text=path.read_text(encoding="utf-8",errors="replace")
|
||||
27 for token in text.replace(chr(10)," ").split():
|
||||
28 if "=" in token:
|
||||
29 key,value=token.split("=",1)
|
||||
30 if key and key[0].isalpha():
|
||||
31 data[key]=value
|
||||
32 return str(data.get("STATUS",data.get("status","UNKNOWN")))
|
||||
33 def normalize(value):
|
||||
34 value=str(value).strip().upper()
|
||||
35 if value in {"OK","PASS","PASSED","SUCCESS","HEALTHY"}:
|
||||
36 return "OK"
|
||||
37 if value in {"WARN","WARNING"}:
|
||||
38 return "WARN"
|
||||
39 if value in {"FAIL","FAILED","ERROR","CRITICAL","BAD"}:
|
||||
40 return "FAIL"
|
||||
41 return value or "UNKNOWN"
|
||||
42 matrix=json.loads(M.read_text(encoding="utf-8"))
|
||||
43 ROOT.mkdir(parents=True,exist_ok=True)
|
||||
44 RENDER.mkdir(parents=True,exist_ok=True)
|
||||
45 COMPAT.mkdir(parents=True,exist_ok=True)
|
||||
46 now=int(time.time())
|
||||
47 now_iso=time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime(now))
|
||||
48 results=[]
|
||||
49 compat_map=[]
|
||||
50 for job in matrix.get("jobs",[]):
|
||||
51 original_sources=list(job.get("evidence_sources_original",[]))
|
||||
52 evidence_inputs=list(job.get("evidence_inputs",[]))
|
||||
53 if original_sources and len(original_sources)!=len(evidence_inputs):
|
||||
54 raise RuntimeError("evidence_mapping_length_mismatch:"+str(job.get("name","")))
|
||||
55 for source_raw,destination_raw in zip(original_sources,evidence_inputs):
|
||||
56 source_path=Path(source_raw)
|
||||
57 destination_path=Path(destination_raw)
|
||||
58 if not source_path.is_file():
|
||||
59 continue
|
||||
60 head=source_path.read_text(encoding="utf-8",errors="ignore")[:4096]
|
||||
61 if "TYPE=backup-matrix-generated" in head or "CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json" in head:
|
||||
62 continue
|
||||
63 destination_path.parent.mkdir(parents=True,exist_ok=True)
|
||||
64 temporary=destination_path.with_name(destination_path.name+".tmp-ingest")
|
||||
65 shutil.copy2(source_path,temporary)
|
||||
66 os.replace(temporary,destination_path)
|
||||
67 jid=str(job.get("id",""))
|
||||
68 name=str(job.get("name",jid))
|
||||
69 timer=str(job.get("timer",""))
|
||||
70 service=str(job.get("service",""))
|
||||
71 enabled=cmd(["systemctl","is-enabled",timer]) if timer else "missing"
|
||||
72 active=cmd(["systemctl","is-active",timer]) if timer else "missing"
|
||||
73 source_kind=str(job.get("source_kind","health_file"))
|
||||
74 best=None
|
||||
75 for raw in job.get("evidence_inputs",job.get("legacy_health_inputs",[])):
|
||||
76 path=Path(raw)
|
||||
77 if not path.is_file():
|
||||
78 continue
|
||||
79 mtime=int(path.stat().st_mtime)
|
||||
80 if best is None or mtime>best[0]:
|
||||
81 best=(mtime,str(path),parse_health(path))
|
||||
82 service_result=cmd(["systemctl","show",service,"-p","Result","--value"]) if service else "missing"
|
||||
83 service_exec_main_status=cmd(["systemctl","show",service,"-p","ExecMainStatus","--value"]) if service else "missing"
|
||||
84 evidence_level="artifact_or_health"
|
||||
85 if best is None and source_kind=="systemd":
|
||||
86 stamp=Path("/var/lib/systemd/timers")/("stamp-"+timer)
|
||||
87 mtime=int(stamp.stat().st_mtime) if stamp.is_file() else 0
|
||||
88 source="systemd:"+service
|
||||
89 source_status="OK" if service_result=="success" and service_exec_main_status=="0" else "FAIL"
|
||||
90 evidence_level="execution_only"
|
||||
91 else:
|
||||
92 source=best[1] if best else ""
|
||||
93 source_status=normalize(best[2]) if best else "UNMAPPED"
|
||||
94 mtime=best[0] if best else 0
|
||||
95 age=now-mtime if mtime else -1
|
||||
96 max_age=int(job.get("max_age_seconds",172800))
|
||||
97 critical=bool(job.get("critical",False))
|
||||
98 if not critical and enabled!="enabled":
|
||||
99 final="DISABLED"
|
||||
100 elif critical and (enabled!="enabled" or active!="active"):
|
||||
101 final="FAIL"
|
||||
102 elif source_status=="UNMAPPED":
|
||||
103 final="UNMAPPED"
|
||||
104 elif source_status!="OK":
|
||||
105 final="FAIL"
|
||||
106 elif age<0:
|
||||
107 final="MISSING"
|
||||
108 elif age>max_age:
|
||||
109 final="STALE"
|
||||
110 else:
|
||||
111 final="OK"
|
||||
112 freshness="UNKNOWN" if age<0 else ("FRESH" if age<=max_age else "STALE")
|
||||
113 source_mtime=time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime(mtime)) if mtime else ""
|
||||
114 result={"id":jid,"name":name,"category":job.get("category","unknown"),"critical":critical,"timer":timer,"service":service,"timer_enabled":enabled,"timer_active":active,"status":final,"source_kind":source_kind,"evidence_level":evidence_level,"service_result":service_result,"service_exec_main_status":service_exec_main_status,"source_health":source,"source_status":source_status,"source_mtime":source_mtime,"age_seconds":age,"max_age_seconds":max_age,"freshness":freshness,"generated_health_output":job.get("generated_health_output","")}
|
||||
115 results.append(result)
|
||||
116 mode_value=matrix.get("mode","shadow"); lines=[f"STATUS={final}",f"TS={now_iso}","TYPE=backup-matrix-generated",f"JOB_ID={jid}",f"JOB_NAME={name}",f"MATRIX_MODE={mode_value}",f"TIMER={timer}",f"SERVICE={service}",f"TIMER_ENABLED={enabled}",f"TIMER_ACTIVE={active}",f"SOURCE_KIND={source_kind}",f"EVIDENCE_LEVEL={evidence_level}",f"SERVICE_RESULT={service_result}",f"SERVICE_EXEC_MAIN_STATUS={service_exec_main_status}",f"SOURCE_HEALTH={source}",f"SOURCE_STATUS={source_status}",f"SOURCE_MTIME_UTC={source_mtime}",f"AGE_SECONDS={age}",f"MAX_AGE_SECONDS={max_age}",f"FRESHNESS={freshness}"]
|
||||
117 atomic(Path(job.get("generated_health_output",str(RENDER/(name+".txt")))),chr(10).join(lines)+chr(10))
|
||||
118 for raw in job.get("compatibility_outputs",[]):
|
||||
119 legacy=str(raw)
|
||||
120 digest=hashlib.sha256(legacy.encode("utf-8")).hexdigest()[:12]
|
||||
121 base_name=Path(legacy).name or "health"
|
||||
122 safe=f"{jid}--{base_name}--{digest}.txt"
|
||||
123 compat_path=COMPAT/safe
|
||||
124 compat_lines=lines+[f"LEGACY_SOURCE_PATH={legacy}",f"CANONICAL_JOB_ID={jid}",f"CANONICAL_STATE={ROOT}/state.json"]
|
||||
125 atomic(compat_path,chr(10).join(compat_lines)+chr(10))
|
||||
126 atomic(Path(legacy),chr(10).join(compat_lines)+chr(10))
|
||||
127 compat_map.append({"job_id":jid,"legacy_path":legacy,"compatibility_path":str(compat_path)})
|
||||
128 bad=sum(1 for item in results if item["status"] in {"FAIL","STALE","MISSING"})
|
||||
129 unmapped=sum(1 for item in results if item["status"]=="UNMAPPED")
|
||||
130 disabled=sum(1 for item in results if item["status"]=="DISABLED")
|
||||
131 ok=sum(1 for item in results if item["status"]=="OK")
|
||||
132 mode=str(matrix.get("mode","shadow"))
|
||||
133 overall="OK" if bad==0 and unmapped==0 else ("WARN" if mode=="shadow" else "FAIL")
|
||||
134 compat_map=sorted(compat_map,key=lambda item:(item["legacy_path"],item["job_id"]))
|
||||
135 atomic(ROOT/"compat-map.json",json.dumps(compat_map,ensure_ascii=False,indent=2,sort_keys=True)+chr(10))
|
||||
136 state={"version":1,"generated_at":now_iso,"matrix_mode":mode,"matrix_sha256":hashlib.sha256(M.read_bytes()).hexdigest(),"summary":{"status":overall,"jobs_total":len(results),"jobs_ok":ok,"jobs_bad":bad,"jobs_unmapped":unmapped,"jobs_disabled":disabled},"jobs":results}
|
||||
137 state["authority"]="canonical"
|
||||
138 state["compatibility_map"]=str(ROOT/"compat-map.json")
|
||||
139 state["summary"]["compatibility_outputs"]=len(compat_map)
|
||||
140 atomic(ROOT/"state.json",json.dumps(state,ensure_ascii=False,indent=2,sort_keys=True)+chr(10))
|
||||
===== FILE /usr/local/sbin/homelab-vm160-remote-backup =====
|
||||
/usr/local/sbin/homelab-vm160-remote-backup: Bourne-Again shell script, ASCII text executable
|
||||
1 #!/usr/bin/env bash
|
||||
2 set -Eeuo pipefail
|
||||
3 export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK
|
||||
4 VMID=160
|
||||
5 SOURCE=root@[PRIVATE_IP]
|
||||
6 TARGET=root@[PRIVATE_IP]
|
||||
7 KH=[SENSITIVE_PATH]
|
||||
8 PARENT=/mnt/staging/vzdump
|
||||
9 PREFIX=vm160-pve02-
|
||||
10 HEALTH=/var/lib/homelab-health/vm160-remote-backup.txt
|
||||
11 LOCK=/run/lock/homelab-vm160-remote-backup.lock
|
||||
12 SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$KH")
|
||||
13 TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker
|
||||
14 target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; }
|
||||
15 install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")"
|
||||
16 exec 9>"$LOCK"
|
||||
17 flock -n 9
|
||||
18 test "$(hostname -s)" = pve01
|
||||
19 test -x "$TARGET_WORKER"
|
||||
20 test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running
|
||||
21 SOURCE_ACTIVE_RC=0
|
||||
22 SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$?
|
||||
23 test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1
|
||||
24 test -z "$SOURCE_ACTIVE"
|
||||
25 TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
26 DIR="$PARENT/${PREFIX}${TS}"
|
||||
27 NAME="vzdump-qemu-${VMID}-${TS}.vma.zst"
|
||||
28 PART="$DIR/.${NAME}.partial"
|
||||
29 FINAL="$DIR/$NAME"
|
||||
30 LOG="$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)"
|
||||
31 cleanup(){
|
||||
32 rc=$?
|
||||
33 trap - EXIT
|
||||
34 if test "$rc" -ne 0; then
|
||||
35 ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true
|
||||
36 tmp="${HEALTH}.tmp"
|
||||
37 printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp"
|
||||
38 chmod 0644 "$tmp"
|
||||
39 mv "$tmp" "$HEALTH"
|
||||
40 fi
|
||||
41 rm -f "$LOG"
|
||||
42 exit "$rc"
|
||||
43 }
|
||||
44 trap cleanup EXIT
|
||||
45 REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")"
|
||||
46 LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")"
|
||||
47 LATEST_SIZE="${LATEST_SIZE:-0}"
|
||||
48 if test "$LATEST_SIZE" -gt 0; then
|
||||
49 REQUIRED="$((LATEST_SIZE + LATEST_SIZE / 4 + 10737418240))"
|
||||
50 else
|
||||
51 REQUIRED=161061273600
|
||||
52 fi
|
||||
53 test "$REMOTE_FREE" -ge "$REQUIRED"
|
||||
54 ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'"
|
||||
55 ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'"
|
||||
56 target_worker finalize "$PARENT" "$PREFIX" "$PART" "$FINAL" "$DIR" "$NAME"
|
||||
57 ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG"
|
||||
58 SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")"
|
||||
59 SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")"
|
||||
60 target_worker promote "$PARENT" "$PREFIX" "$DIR" "$TS"
|
||||
61 tmp="${HEALTH}.tmp"
|
||||
62 printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp"
|
||||
63 chmod 0644 "$tmp"
|
||||
64 mv "$tmp" "$HEALTH"
|
||||
65 trap - EXIT
|
||||
66 rm -f "$LOG"
|
||||
67 printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA"
|
||||
===== FILE /usr/local/sbin/homelab-cluster-admin-pull =====
|
||||
/usr/local/sbin/homelab-cluster-admin-pull: Bourne-Again shell script, ASCII text executable, with very long lines (424)
|
||||
1 #!/usr/bin/env bash
|
||||
2 set +e +u +o pipefail 2>/dev/null || true
|
||||
3 TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
4 ROOT=/var/lib/homelab-cluster-admin
|
||||
5 HEALTH_ROOT=/var/lib/homelab-health
|
||||
6 mkdir -p "$ROOT" "$HEALTH_ROOT"
|
||||
7 REMOTE=$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf "__OBS__\\n"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf "__FULL__\\n"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err)
|
||||
8 RC=$?
|
||||
9 OBS=$(printf "%s\n" "$REMOTE" | awk '/^__OBS__$/{f=1;next}/^__FULL__$/{f=0}f')
|
||||
10 FULL=$(printf "%s\n" "$REMOTE" | awk '/^__FULL__$/{f=1;next}f')
|
||||
11 OBS_BAD=0; FULL_BAD=0
|
||||
12 printf "%s\n" "$OBS" | grep -q "^STATUS=OK" || OBS_BAD=1
|
||||
13 printf "%s\n" "$FULL" | grep -q "^STATUS=OK" || FULL_BAD=1
|
||||
14 printf "%s\n" "$FULL" | grep -q "BAD=0" || FULL_BAD=1
|
||||
15 PROBE_OK=$(printf "%s\n" "$FULL" | sed -n 's/.*PROBE_OK=\([0-9][0-9]*\).*/\1/p' | head -n1)
|
||||
16 PROBE_BAD=$(printf "%s\n" "$FULL" | sed -n 's/.*PROBE_BAD=\([0-9][0-9]*\).*/\1/p' | head -n1)
|
||||
17 [ "${PROBE_BAD:-1}" = 0 ] || FULL_BAD=1
|
||||
18 BAD=$((OBS_BAD+FULL_BAD)); [ "$RC" -eq 0 ] || BAD=$((BAD+1))
|
||||
19 printf "%s\n" "$OBS" > "$ROOT/observer-remote-latest.txt"
|
||||
20 printf "%s\n" "$FULL" > "$ROOT/full-observer-central-latest.md"
|
||||
21 OBS_STATUS=$([ "$OBS_BAD" -eq 0 ] && [ "$RC" -eq 0 ] && echo OK || echo WARN)
|
||||
22 FULL_STATUS=$([ "$FULL_BAD" -eq 0 ] && [ "$RC" -eq 0 ] && echo OK || echo WARN)
|
||||
23 echo "STATUS=$OBS_STATUS TS=$TS TYPE=cluster-admin-central-pull BAD=$OBS_BAD VMID=180 IP=[PRIVATE_IP] SSH_RC=$RC OBSERVER_STATUS=$OBS_STATUS REPORT=$ROOT/observer-remote-latest.txt" > "$HEALTH_ROOT/cluster-admin-observer.txt"
|
||||
24 echo "STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-full-observer-central BAD=$FULL_BAD WARN=0 VMID=180 PULL_RC=$RC FULL_STATUS=$FULL_STATUS PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} REPORT=$ROOT/full-observer-central-latest.md" > "$HEALTH_ROOT/cluster-admin-full-observer.txt"
|
||||
25 echo "STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-restricted-probes BAD=$FULL_BAD VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer" > "$HEALTH_ROOT/cluster-admin-restricted-probes.txt"
|
||||
26 cat "$HEALTH_ROOT/cluster-admin-observer.txt"
|
||||
27 cat "$HEALTH_ROOT/cluster-admin-full-observer.txt"
|
||||
28 cat "$HEALTH_ROOT/cluster-admin-restricted-probes.txt"
|
||||
29 [ "$BAD" -eq 0 ]
|
||||
=== VM160_CONTROL_FLOW ===
|
||||
5:SOURCE=root@[PRIVATE_IP]
|
||||
6:TARGET=root@[PRIVATE_IP]
|
||||
7:KH=[SENSITIVE_PATH]
|
||||
8:PARENT=/mnt/staging/vzdump
|
||||
11:LOCK=/run/lock/homelab-vm160-remote-backup.lock
|
||||
13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker
|
||||
14:target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; }
|
||||
15:install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")"
|
||||
16:exec 9>"$LOCK"
|
||||
19:test -x "$TARGET_WORKER"
|
||||
20:test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running
|
||||
21:SOURCE_ACTIVE_RC=0
|
||||
22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$?
|
||||
23:test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1
|
||||
24:test -z "$SOURCE_ACTIVE"
|
||||
27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst"
|
||||
35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true
|
||||
37: printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp"
|
||||
44:trap cleanup EXIT
|
||||
45:REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")"
|
||||
46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")"
|
||||
48:if test "$LATEST_SIZE" -gt 0; then
|
||||
50:else
|
||||
52:fi
|
||||
53:test "$REMOTE_FREE" -ge "$REQUIRED"
|
||||
54:ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'"
|
||||
55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'"
|
||||
57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG"
|
||||
58:SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")"
|
||||
59:SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")"
|
||||
62:printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp"
|
||||
65:trap - EXIT
|
||||
67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA"
|
||||
=== CLUSTER_ADMIN_CURRENT_FAILURE ===
|
||||
/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
|
||||
/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
|
||||
CHANGES_MADE=NO
|
||||
|
||||
OUTPUT_END
|
||||
|
||||
Reference in New Issue
Block a user