homelab-runtime-publisher
85 lines
4.7 KiB
Plaintext
85 lines
4.7 KiB
Plaintext
CHAT_OUTPUT_BEGIN
|
|
COMMAND_ID=BACKUP-FAILED-UNITS-DECISION-218
|
|
STATUS=OK
|
|
RC=0
|
|
HOST=pve01
|
|
MODE=read-only
|
|
COMPONENT=cluster-audit
|
|
REFERENCE_REGISTER_CHECK=OK
|
|
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
|
|
ERROR_REGISTER_CHECK=OK
|
|
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
|
|
COMMAND_SHA256=67a6392a4f9bb734608ebdf3117c0689e1f36790519a3cf7eaf9a96e738e0730
|
|
DUPLICATE_FAILED_COMMAND_BLOCKED=false
|
|
EXECUTION_STARTED=true
|
|
CHANGES_MADE=false
|
|
SANITIZED=yes
|
|
SECRETS_INCLUDED=no
|
|
PRIVATE_ADDRESSES_INCLUDED=no
|
|
RAW_EVIDENCE_SHA256=ae5182193c2d1cf355898b69c6768e159bf9399e8d682d86ac279d9322128fe1
|
|
SANITIZED_OUTPUT_SHA256=d79dc8b5bff23814516b5d4bc20f905e00c7fe935042a1950f956bb015e5f733
|
|
OUTPUT_BEGIN
|
|
PREVIOUS_COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217
|
|
=== LEDGER_217 ===
|
|
2026-07-22T15:35:58Z BACKUP-FAILED-UNITS-SOURCE-217 2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86 0 OK cluster-audit
|
|
=== MATRIX_ENTRYPOINT ===
|
|
1 #!/usr/bin/env python3
|
|
2 from pathlib import Path
|
|
3 import shutil
|
|
4 import json,subprocess,os,time,sys,hashlib
|
|
5 M=Path("/etc/homelab-backup-matrix/matrix.json")
|
|
6 ROOT=Path("/var/lib/homelab-backup-matrix")
|
|
7 RENDER=ROOT/"rendered"
|
|
8 COMPAT=ROOT/"compat"
|
|
9 def cmd(args):
|
|
10 p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)
|
|
11 return p.stdout.strip() or "unknown"
|
|
12 def atomic(path,text,mode=0o640):
|
|
13 path.parent.mkdir(parents=True,exist_ok=True)
|
|
14 tmp=path.with_name(path.name+".tmp")
|
|
15 tmp.write_text(text,encoding="utf-8")
|
|
16 os.chmod(tmp,mode)
|
|
17 os.replace(tmp,path)
|
|
18 def parse_health(path):
|
|
=== VM160_CONTROL_FLOW ===
|
|
5:SOURCE=root@[PRIVATE_IP]
|
|
6:TARGET=root@[PRIVATE_IP]
|
|
7:KH=[SENSITIVE_PATH]
|
|
8:PARENT=/mnt/staging/vzdump
|
|
11:LOCK=/run/lock/homelab-vm160-remote-backup.lock
|
|
13:TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker
|
|
14:target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; }
|
|
15:install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")"
|
|
16:exec 9>"$LOCK"
|
|
19:test -x "$TARGET_WORKER"
|
|
20:test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running
|
|
21:SOURCE_ACTIVE_RC=0
|
|
22:SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$?
|
|
23:test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1
|
|
24:test -z "$SOURCE_ACTIVE"
|
|
27:NAME="vzdump-qemu-${VMID}-${TS}.vma.zst"
|
|
35: ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true
|
|
37: printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp"
|
|
44:trap cleanup EXIT
|
|
45:REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")"
|
|
46:LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")"
|
|
48:if test "$LATEST_SIZE" -gt 0; then
|
|
50:else
|
|
52:fi
|
|
53:test "$REMOTE_FREE" -ge "$REQUIRED"
|
|
54:ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'"
|
|
55:ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'"
|
|
57:ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG"
|
|
58:SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")"
|
|
59:SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")"
|
|
62:printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp"
|
|
65:trap - EXIT
|
|
67:printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA"
|
|
=== CLUSTER_ADMIN_CURRENT_FAILURE ===
|
|
/var/lib/homelab-cluster-admin/observer-remote-latest.txt:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
|
|
/var/lib/homelab-cluster-admin/full-observer-central-latest.md:1:STATUS=WARN TS=2026-07-20T23:36:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
|
|
CHANGES_MADE=NO
|
|
|
|
OUTPUT_END
|
|
CHAT_OUTPUT_END
|