CHAT_OUTPUT_BEGIN COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217 STATUS=OK RC=0 HOST=pve01 MODE=read-only COMPONENT=cluster-audit REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 COMMAND_SHA256=2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no RAW_EVIDENCE_SHA256=96e72a8cec4b5e5cde52a48e6816f4ce162311b07530ee398c57efc67f3480b1 SANITIZED_OUTPUT_SHA256=dbe6bb8b0419312d9774755ff8668c435118bc102b49b0255667d04f06dda499 OUTPUT_BEGIN PREVIOUS_COMMAND_ID=RUNTIME-PUBLISH-GAP-DIAG-216 ===== UNIT homelab-backup-matrix-refresh ===== # /etc/systemd/system/homelab-backup-matrix-refresh.service [Unit] Description=Generate unified homelab backup matrix state After=local-fs.target [Service] Type=oneshot ExecStart=/usr/local/sbin/homelab-backup-matrix-refresh TimeoutStartSec=5min ===== UNIT homelab-vm160-remote-backup ===== # /etc/systemd/system/homelab-vm160-remote-backup.service [Unit] Description=VM160 full-image backup streamed from pve02 to pve03 After=network-online.target pve-cluster.service Wants=network-online.target [Service] Type=oneshot TimeoutStartSec=6h ExecStart=/usr/local/sbin/homelab-vm160-remote-backup ===== UNIT homelab-cluster-admin-pull ===== # /etc/systemd/system/homelab-cluster-admin-pull.service [Unit] Description=Pull cluster-admin observer health into pve01 [Service] Type=oneshot ExecStart=/usr/local/sbin/homelab-cluster-admin-pull ===== FILE /usr/local/sbin/homelab-backup-matrix-refresh ===== /usr/local/sbin/homelab-backup-matrix-refresh: Python script, ASCII text executable, with very long lines (592) 1 #!/usr/bin/env python3 2 from pathlib import Path 3 import shutil 4 import json,subprocess,os,time,sys,hashlib 5 M=Path("/etc/homelab-backup-matrix/matrix.json") 6 ROOT=Path("/var/lib/homelab-backup-matrix") 7 RENDER=ROOT/"rendered" 8 COMPAT=ROOT/"compat" 9 def cmd(args): 10 p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False) 11 return p.stdout.strip() or "unknown" 12 def atomic(path,text,mode=0o640): 13 path.parent.mkdir(parents=True,exist_ok=True) 14 tmp=path.with_name(path.name+".tmp") 15 tmp.write_text(text,encoding="utf-8") 16 os.chmod(tmp,mode) 17 os.replace(tmp,path) 18 def parse_health(path): 19 if path.suffix.lower()==".json": 20 try: 21 data=json.loads(path.read_text(encoding="utf-8",errors="replace")) 22 return str(data.get("status",data.get("STATUS",data.get("result","UNKNOWN")))) 23 except Exception: 24 return "ERROR" 25 data={} 26 text=path.read_text(encoding="utf-8",errors="replace") 27 for token in text.replace(chr(10)," ").split(): 28 if "=" in token: 29 key,value=token.split("=",1) 30 if key and key[0].isalpha(): 31 data[key]=value 32 return str(data.get("STATUS",data.get("status","UNKNOWN"))) 33 def normalize(value): 34 value=str(value).strip().upper() 35 if value in {"OK","PASS","PASSED","SUCCESS","HEALTHY"}: 36 return "OK" 37 if value in {"WARN","WARNING"}: 38 return "WARN" 39 if value in {"FAIL","FAILED","ERROR","CRITICAL","BAD"}: 40 return "FAIL" 41 return value or "UNKNOWN" 42 matrix=json.loads(M.read_text(encoding="utf-8")) 43 ROOT.mkdir(parents=True,exist_ok=True) 44 RENDER.mkdir(parents=True,exist_ok=True) 45 COMPAT.mkdir(parents=True,exist_ok=True) 46 now=int(time.time()) 47 now_iso=time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime(now)) 48 results=[] 49 compat_map=[] 50 for job in matrix.get("jobs",[]): 51 original_sources=list(job.get("evidence_sources_original",[])) 52 evidence_inputs=list(job.get("evidence_inputs",[])) 53 if original_sources and len(original_sources)!=len(evidence_inputs): 54 raise RuntimeError("evidence_mapping_length_mismatch:"+str(job.get("name",""))) 55 for source_raw,destination_raw in zip(original_sources,evidence_inputs): 56 source_path=Path(source_raw) 57 destination_path=Path(destination_raw) 58 if not source_path.is_file(): 59 continue 60 head=source_path.read_text(encoding="utf-8",errors="ignore")[:4096] 61 if "TYPE=backup-matrix-generated" in head or "CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json" in head: 62 continue 63 destination_path.parent.mkdir(parents=True,exist_ok=True) 64 temporary=destination_path.with_name(destination_path.name+".tmp-ingest") 65 shutil.copy2(source_path,temporary) 66 os.replace(temporary,destination_path) 67 jid=str(job.get("id","")) 68 name=str(job.get("name",jid)) 69 timer=str(job.get("timer","")) 70 service=str(job.get("service","")) 71 enabled=cmd(["systemctl","is-enabled",timer]) if timer else "missing" 72 active=cmd(["systemctl","is-active",timer]) if timer else "missing" 73 source_kind=str(job.get("source_kind","health_file")) 74 best=None 75 for raw in job.get("evidence_inputs",job.get("legacy_health_inputs",[])): 76 path=Path(raw) 77 if not path.is_file(): 78 continue 79 mtime=int(path.stat().st_mtime) 80 if best is None or mtime>best[0]: 81 best=(mtime,str(path),parse_health(path)) 82 service_result=cmd(["systemctl","show",service,"-p","Result","--value"]) if service else "missing" 83 service_exec_main_status=cmd(["systemctl","show",service,"-p","ExecMainStatus","--value"]) if service else "missing" 84 evidence_level="artifact_or_health" 85 if best is None and source_kind=="systemd": 86 stamp=Path("/var/lib/systemd/timers")/("stamp-"+timer) 87 mtime=int(stamp.stat().st_mtime) if stamp.is_file() else 0 88 source="systemd:"+service 89 source_status="OK" if service_result=="success" and service_exec_main_status=="0" else "FAIL" 90 evidence_level="execution_only" 91 else: 92 source=best[1] if best else "" 93 source_status=normalize(best[2]) if best else "UNMAPPED" 94 mtime=best[0] if best else 0 95 age=now-mtime if mtime else -1 96 max_age=int(job.get("max_age_seconds",172800)) 97 critical=bool(job.get("critical",False)) 98 if not critical and enabled!="enabled": 99 final="DISABLED" 100 elif critical and (enabled!="enabled" or active!="active"): 101 final="FAIL" 102 elif source_status=="UNMAPPED": 103 final="UNMAPPED" 104 elif source_status!="OK": 105 final="FAIL" 106 elif age<0: 107 final="MISSING" 108 elif age>max_age: 109 final="STALE" 110 else: 111 final="OK" 112 freshness="UNKNOWN" if age<0 else ("FRESH" if age<=max_age else "STALE") 113 source_mtime=time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime(mtime)) if mtime else "" 114 result={"id":jid,"name":name,"category":job.get("category","unknown"),"critical":critical,"timer":timer,"service":service,"timer_enabled":enabled,"timer_active":active,"status":final,"source_kind":source_kind,"evidence_level":evidence_level,"service_result":service_result,"service_exec_main_status":service_exec_main_status,"source_health":source,"source_status":source_status,"source_mtime":source_mtime,"age_seconds":age,"max_age_seconds":max_age,"freshness":freshness,"generated_health_output":job.get("generated_health_output","")} 115 results.append(result) 116 mode_value=matrix.get("mode","shadow"); lines=[f"STATUS={final}",f"TS={now_iso}","TYPE=backup-matrix-generated",f"JOB_ID={jid}",f"JOB_NAME={name}",f"MATRIX_MODE={mode_value}",f"TIMER={timer}",f"SERVICE={service}",f"TIMER_ENABLED={enabled}",f"TIMER_ACTIVE={active}",f"SOURCE_KIND={source_kind}",f"EVIDENCE_LEVEL={evidence_level}",f"SERVICE_RESULT={service_result}",f"SERVICE_EXEC_MAIN_STATUS={service_exec_main_status}",f"SOURCE_HEALTH={source}",f"SOURCE_STATUS={source_status}",f"SOURCE_MTIME_UTC={source_mtime}",f"AGE_SECONDS={age}",f"MAX_AGE_SECONDS={max_age}",f"FRESHNESS={freshness}"] 117 atomic(Path(job.get("generated_health_output",str(RENDER/(name+".txt")))),chr(10).join(lines)+chr(10)) 118 for raw in job.get("compatibility_outputs",[]): 119 legacy=str(raw) 120 digest=hashlib.sha256(legacy.encode("utf-8")).hexdigest()[:12] 121 base_name=Path(legacy).name or "health" 122 safe=f"{jid}--{base_name}--{digest}.txt" 123 compat_path=COMPAT/safe 124 compat_lines=lines+[f"LEGACY_SOURCE_PATH={legacy}",f"CANONICAL_JOB_ID={jid}",f"CANONICAL_STATE={ROOT}/state.json"] 125 atomic(compat_path,chr(10).join(compat_lines)+chr(10)) 126 atomic(Path(legacy),chr(10).join(compat_lines)+chr(10)) 127 compat_map.append({"job_id":jid,"legacy_path":legacy,"compatibility_path":str(compat_path)}) 128 bad=sum(1 for item in results if item["status"] in {"FAIL","STALE","MISSING"}) 129 unmapped=sum(1 for item in results if item["status"]=="UNMAPPED") 130 disabled=sum(1 for item in results if item["status"]=="DISABLED") 131 ok=sum(1 for item in results if item["status"]=="OK") 132 mode=str(matrix.get("mode","shadow")) 133 overall="OK" if bad==0 and unmapped==0 else ("WARN" if mode=="shadow" else "FAIL") 134 compat_map=sorted(compat_map,key=lambda item:(item["legacy_path"],item["job_id"])) 135 atomic(ROOT/"compat-map.json",json.dumps(compat_map,ensure_ascii=False,indent=2,sort_keys=True)+chr(10)) 136 state={"version":1,"generated_at":now_iso,"matrix_mode":mode,"matrix_sha256":hashlib.sha256(M.read_bytes()).hexdigest(),"summary":{"status":overall,"jobs_total":len(results),"jobs_ok":ok,"jobs_bad":bad,"jobs_unmapped":unmapped,"jobs_disabled":disabled},"jobs":results} 137 state["authority"]="canonical" 138 state["compatibility_map"]=str(ROOT/"compat-map.json") 139 state["summary"]["compatibility_outputs"]=len(compat_map) 140 atomic(ROOT/"state.json",json.dumps(state,ensure_ascii=False,indent=2,sort_keys=True)+chr(10)) ===== FILE /usr/local/sbin/homelab-vm160-remote-backup ===== /usr/local/sbin/homelab-vm160-remote-backup: Bourne-Again shell script, ASCII text executable 1 #!/usr/bin/env bash 2 set -Eeuo pipefail 3 export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK 4 VMID=160 5 SOURCE=root@[PRIVATE_IP] 6 TARGET=root@[PRIVATE_IP] 7 KH=[SENSITIVE_PATH] 8 PARENT=/mnt/staging/vzdump 9 PREFIX=vm160-pve02- 10 HEALTH=/var/lib/homelab-health/vm160-remote-backup.txt 11 LOCK=/run/lock/homelab-vm160-remote-backup.lock 12 SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$KH") 13 TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker 14 target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; } 15 install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")" 16 exec 9>"$LOCK" 17 flock -n 9 18 test "$(hostname -s)" = pve01 19 test -x "$TARGET_WORKER" 20 test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running 21 SOURCE_ACTIVE_RC=0 22 SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$? 23 test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1 24 test -z "$SOURCE_ACTIVE" 25 TS="$(date -u +%Y%m%dT%H%M%SZ)" 26 DIR="$PARENT/${PREFIX}${TS}" 27 NAME="vzdump-qemu-${VMID}-${TS}.vma.zst" 28 PART="$DIR/.${NAME}.partial" 29 FINAL="$DIR/$NAME" 30 LOG="$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)" 31 cleanup(){ 32 rc=$? 33 trap - EXIT 34 if test "$rc" -ne 0; then 35 ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true 36 tmp="${HEALTH}.tmp" 37 printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp" 38 chmod 0644 "$tmp" 39 mv "$tmp" "$HEALTH" 40 fi 41 rm -f "$LOG" 42 exit "$rc" 43 } 44 trap cleanup EXIT 45 REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")" 46 LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")" 47 LATEST_SIZE="${LATEST_SIZE:-0}" 48 if test "$LATEST_SIZE" -gt 0; then 49 REQUIRED="$((LATEST_SIZE + LATEST_SIZE / 4 + 10737418240))" 50 else 51 REQUIRED=161061273600 52 fi 53 test "$REMOTE_FREE" -ge "$REQUIRED" 54 ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'" 55 ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'" 56 target_worker finalize "$PARENT" "$PREFIX" "$PART" "$FINAL" "$DIR" "$NAME" 57 ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG" 58 SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")" 59 SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")" 60 target_worker promote "$PARENT" "$PREFIX" "$DIR" "$TS" 61 tmp="${HEALTH}.tmp" 62 printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp" 63 chmod 0644 "$tmp" 64 mv "$tmp" "$HEALTH" 65 trap - EXIT 66 rm -f "$LOG" 67 printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA" ===== FILE /usr/local/sbin/homelab-cluster-admin-pull ===== /usr/local/sbin/homelab-cluster-admin-pull: Bourne-Again shell script, ASCII text executable, with very long lines (424) 1 #!/usr/bin/env bash 2 set +e +u +o pipefail 2>/dev/null || true 3 TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) 4 ROOT=/var/lib/homelab-cluster-admin 5 HEALTH_ROOT=/var/lib/homelab-health 6 mkdir -p "$ROOT" "$HEALTH_ROOT" 7 REMOTE=$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf "__OBS__\\n"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf "__FULL__\\n"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err) 8 RC=$? 9 OBS=$(printf "%s\n" "$REMOTE" | awk '/^__OBS__$/{f=1;next}/^__FULL__$/{f=0}f') 10 FULL=$(printf "%s\n" "$REMOTE" | awk '/^__FULL__$/{f=1;next}f') 11 OBS_BAD=0; FULL_BAD=0 12 printf "%s\n" "$OBS" | grep -q "^STATUS=OK" || OBS_BAD=1 13 printf "%s\n" "$FULL" | grep -q "^STATUS=OK" || FULL_BAD=1 14 printf "%s\n" "$FULL" | grep -q "BAD=0" || FULL_BAD=1 15 PROBE_OK=$(printf "%s\n" "$FULL" | sed -n 's/.*PROBE_OK=\([0-9][0-9]*\).*/\1/p' | head -n1) 16 PROBE_BAD=$(printf "%s\n" "$FULL" | sed -n 's/.*PROBE_BAD=\([0-9][0-9]*\).*/\1/p' | head -n1) 17 [ "${PROBE_BAD:-1}" = 0 ] || FULL_BAD=1 18 BAD=$((OBS_BAD+FULL_BAD)); [ "$RC" -eq 0 ] || BAD=$((BAD+1)) 19 printf "%s\n" "$OBS" > "$ROOT/observer-remote-latest.txt" 20 printf "%s\n" "$FULL" > "$ROOT/full-observer-central-latest.md" 21 OBS_STATUS=$([ "$OBS_BAD" -eq 0 ] && [ "$RC" -eq 0 ] && echo OK || echo WARN) 22 FULL_STATUS=$([ "$FULL_BAD" -eq 0 ] && [ "$RC" -eq 0 ] && echo OK || echo WARN) 23 echo "STATUS=$OBS_STATUS TS=$TS TYPE=cluster-admin-central-pull BAD=$OBS_BAD VMID=180 IP=[PRIVATE_IP] SSH_RC=$RC OBSERVER_STATUS=$OBS_STATUS REPORT=$ROOT/observer-remote-latest.txt" > "$HEALTH_ROOT/cluster-admin-observer.txt" 24 echo "STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-full-observer-central BAD=$FULL_BAD WARN=0 VMID=180 PULL_RC=$RC FULL_STATUS=$FULL_STATUS PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} REPORT=$ROOT/full-observer-central-latest.md" > "$HEALTH_ROOT/cluster-admin-full-observer.txt" 25 echo "STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-restricted-probes BAD=$FULL_BAD VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer" > "$HEALTH_ROOT/cluster-admin-restricted-probes.txt" 26 cat "$HEALTH_ROOT/cluster-admin-observer.txt" 27 cat "$HEALTH_ROOT/cluster-admin-full-observer.txt" 28 cat "$HEALTH_ROOT/cluster-admin-restricted-probes.txt" 29 [ "$BAD" -eq 0 ] CHANGES_MADE=NO OUTPUT_END CHAT_OUTPUT_END