CHAT_OUTPUT_BEGIN
COMMAND_ID=BACKUP-FAILED-UNITS-SOURCE-217
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=96e72a8cec4b5e5cde52a48e6816f4ce162311b07530ee398c57efc67f3480b1
SANITIZED_OUTPUT_SHA256=dbe6bb8b0419312d9774755ff8668c435118bc102b49b0255667d04f06dda499
OUTPUT_BEGIN
PREVIOUS_COMMAND_ID=RUNTIME-PUBLISH-GAP-DIAG-216
===== UNIT homelab-backup-matrix-refresh =====
# /etc/systemd/system/homelab-backup-matrix-refresh.service
[Unit]
Description=Generate unified homelab backup matrix state
After=local-fs.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-backup-matrix-refresh
TimeoutStartSec=5min
===== UNIT homelab-vm160-remote-backup =====
# /etc/systemd/system/homelab-vm160-remote-backup.service
[Unit]
Description=VM160 full-image backup streamed from pve02 to pve03
After=network-online.target pve-cluster.service
Wants=network-online.target

[Service]
Type=oneshot
TimeoutStartSec=6h
ExecStart=/usr/local/sbin/homelab-vm160-remote-backup
===== UNIT homelab-cluster-admin-pull =====
# /etc/systemd/system/homelab-cluster-admin-pull.service
[Unit]
Description=Pull cluster-admin observer health into pve01

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-cluster-admin-pull
===== FILE /usr/local/sbin/homelab-backup-matrix-refresh =====
/usr/local/sbin/homelab-backup-matrix-refresh: Python script, ASCII text executable, with very long lines (592)
     1	#!/usr/bin/env python3
     2	from pathlib import Path
     3	import shutil
     4	import json,subprocess,os,time,sys,hashlib
     5	M=Path("/etc/homelab-backup-matrix/matrix.json")
     6	ROOT=Path("/var/lib/homelab-backup-matrix")
     7	RENDER=ROOT/"rendered"
     8	COMPAT=ROOT/"compat"
     9	def cmd(args):
    10	    p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)
    11	    return p.stdout.strip() or "unknown"
    12	def atomic(path,text,mode=0o640):
    13	    path.parent.mkdir(parents=True,exist_ok=True)
    14	    tmp=path.with_name(path.name+".tmp")
    15	    tmp.write_text(text,encoding="utf-8")
    16	    os.chmod(tmp,mode)
    17	    os.replace(tmp,path)
    18	def parse_health(path):
    19	    if path.suffix.lower()==".json":
    20	        try:
    21	            data=json.loads(path.read_text(encoding="utf-8",errors="replace"))
    22	            return str(data.get("status",data.get("STATUS",data.get("result","UNKNOWN"))))
    23	        except Exception:
    24	            return "ERROR"
    25	    data={}
    26	    text=path.read_text(encoding="utf-8",errors="replace")
    27	    for token in text.replace(chr(10)," ").split():
    28	        if "=" in token:
    29	            key,value=token.split("=",1)
    30	            if key and key[0].isalpha():
    31	                data[key]=value
    32	    return str(data.get("STATUS",data.get("status","UNKNOWN")))
    33	def normalize(value):
    34	    value=str(value).strip().upper()
    35	    if value in {"OK","PASS","PASSED","SUCCESS","HEALTHY"}:
    36	        return "OK"
    37	    if value in {"WARN","WARNING"}:
    38	        return "WARN"
    39	    if value in {"FAIL","FAILED","ERROR","CRITICAL","BAD"}:
    40	        return "FAIL"
    41	    return value or "UNKNOWN"
    42	matrix=json.loads(M.read_text(encoding="utf-8"))
    43	ROOT.mkdir(parents=True,exist_ok=True)
    44	RENDER.mkdir(parents=True,exist_ok=True)
    45	COMPAT.mkdir(parents=True,exist_ok=True)
    46	now=int(time.time())
    47	now_iso=time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime(now))
    48	results=[]
    49	compat_map=[]
    50	for job in matrix.get("jobs",[]):
    51	    original_sources=list(job.get("evidence_sources_original",[]))
    52	    evidence_inputs=list(job.get("evidence_inputs",[]))
    53	    if original_sources and len(original_sources)!=len(evidence_inputs):
    54	        raise RuntimeError("evidence_mapping_length_mismatch:"+str(job.get("name","")))
    55	    for source_raw,destination_raw in zip(original_sources,evidence_inputs):
    56	        source_path=Path(source_raw)
    57	        destination_path=Path(destination_raw)
    58	        if not source_path.is_file():
    59	            continue
    60	        head=source_path.read_text(encoding="utf-8",errors="ignore")[:4096]
    61	        if "TYPE=backup-matrix-generated" in head or "CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json" in head:
    62	            continue
    63	        destination_path.parent.mkdir(parents=True,exist_ok=True)
    64	        temporary=destination_path.with_name(destination_path.name+".tmp-ingest")
    65	        shutil.copy2(source_path,temporary)
    66	        os.replace(temporary,destination_path)
    67	    jid=str(job.get("id",""))
    68	    name=str(job.get("name",jid))
    69	    timer=str(job.get("timer",""))
    70	    service=str(job.get("service",""))
    71	    enabled=cmd(["systemctl","is-enabled",timer]) if timer else "missing"
    72	    active=cmd(["systemctl","is-active",timer]) if timer else "missing"
    73	    source_kind=str(job.get("source_kind","health_file"))
    74	    best=None
    75	    for raw in job.get("evidence_inputs",job.get("legacy_health_inputs",[])):
    76	        path=Path(raw)
    77	        if not path.is_file():
    78	            continue
    79	        mtime=int(path.stat().st_mtime)
    80	        if best is None or mtime>best[0]:
    81	            best=(mtime,str(path),parse_health(path))
    82	    service_result=cmd(["systemctl","show",service,"-p","Result","--value"]) if service else "missing"
    83	    service_exec_main_status=cmd(["systemctl","show",service,"-p","ExecMainStatus","--value"]) if service else "missing"
    84	    evidence_level="artifact_or_health"
    85	    if best is None and source_kind=="systemd":
    86	        stamp=Path("/var/lib/systemd/timers")/("stamp-"+timer)
    87	        mtime=int(stamp.stat().st_mtime) if stamp.is_file() else 0
    88	        source="systemd:"+service
    89	        source_status="OK" if service_result=="success" and service_exec_main_status=="0" else "FAIL"
    90	        evidence_level="execution_only"
    91	    else:
    92	        source=best[1] if best else ""
    93	        source_status=normalize(best[2]) if best else "UNMAPPED"
    94	        mtime=best[0] if best else 0
    95	    age=now-mtime if mtime else -1
    96	    max_age=int(job.get("max_age_seconds",172800))
    97	    critical=bool(job.get("critical",False))
    98	    if not critical and enabled!="enabled":
    99	        final="DISABLED"
   100	    elif critical and (enabled!="enabled" or active!="active"):
   101	        final="FAIL"
   102	    elif source_status=="UNMAPPED":
   103	        final="UNMAPPED"
   104	    elif source_status!="OK":
   105	        final="FAIL"
   106	    elif age<0:
   107	        final="MISSING"
   108	    elif age>max_age:
   109	        final="STALE"
   110	    else:
   111	        final="OK"
   112	    freshness="UNKNOWN" if age<0 else ("FRESH" if age<=max_age else "STALE")
   113	    source_mtime=time.strftime("%Y-%m-%dT%H:%M:%SZ",time.gmtime(mtime)) if mtime else ""
   114	    result={"id":jid,"name":name,"category":job.get("category","unknown"),"critical":critical,"timer":timer,"service":service,"timer_enabled":enabled,"timer_active":active,"status":final,"source_kind":source_kind,"evidence_level":evidence_level,"service_result":service_result,"service_exec_main_status":service_exec_main_status,"source_health":source,"source_status":source_status,"source_mtime":source_mtime,"age_seconds":age,"max_age_seconds":max_age,"freshness":freshness,"generated_health_output":job.get("generated_health_output","")}
   115	    results.append(result)
   116	    mode_value=matrix.get("mode","shadow"); lines=[f"STATUS={final}",f"TS={now_iso}","TYPE=backup-matrix-generated",f"JOB_ID={jid}",f"JOB_NAME={name}",f"MATRIX_MODE={mode_value}",f"TIMER={timer}",f"SERVICE={service}",f"TIMER_ENABLED={enabled}",f"TIMER_ACTIVE={active}",f"SOURCE_KIND={source_kind}",f"EVIDENCE_LEVEL={evidence_level}",f"SERVICE_RESULT={service_result}",f"SERVICE_EXEC_MAIN_STATUS={service_exec_main_status}",f"SOURCE_HEALTH={source}",f"SOURCE_STATUS={source_status}",f"SOURCE_MTIME_UTC={source_mtime}",f"AGE_SECONDS={age}",f"MAX_AGE_SECONDS={max_age}",f"FRESHNESS={freshness}"]
   117	    atomic(Path(job.get("generated_health_output",str(RENDER/(name+".txt")))),chr(10).join(lines)+chr(10))
   118	    for raw in job.get("compatibility_outputs",[]):
   119	        legacy=str(raw)
   120	        digest=hashlib.sha256(legacy.encode("utf-8")).hexdigest()[:12]
   121	        base_name=Path(legacy).name or "health"
   122	        safe=f"{jid}--{base_name}--{digest}.txt"
   123	        compat_path=COMPAT/safe
   124	        compat_lines=lines+[f"LEGACY_SOURCE_PATH={legacy}",f"CANONICAL_JOB_ID={jid}",f"CANONICAL_STATE={ROOT}/state.json"]
   125	        atomic(compat_path,chr(10).join(compat_lines)+chr(10))
   126	        atomic(Path(legacy),chr(10).join(compat_lines)+chr(10))
   127	        compat_map.append({"job_id":jid,"legacy_path":legacy,"compatibility_path":str(compat_path)})
   128	bad=sum(1 for item in results if item["status"] in {"FAIL","STALE","MISSING"})
   129	unmapped=sum(1 for item in results if item["status"]=="UNMAPPED")
   130	disabled=sum(1 for item in results if item["status"]=="DISABLED")
   131	ok=sum(1 for item in results if item["status"]=="OK")
   132	mode=str(matrix.get("mode","shadow"))
   133	overall="OK" if bad==0 and unmapped==0 else ("WARN" if mode=="shadow" else "FAIL")
   134	compat_map=sorted(compat_map,key=lambda item:(item["legacy_path"],item["job_id"]))
   135	atomic(ROOT/"compat-map.json",json.dumps(compat_map,ensure_ascii=False,indent=2,sort_keys=True)+chr(10))
   136	state={"version":1,"generated_at":now_iso,"matrix_mode":mode,"matrix_sha256":hashlib.sha256(M.read_bytes()).hexdigest(),"summary":{"status":overall,"jobs_total":len(results),"jobs_ok":ok,"jobs_bad":bad,"jobs_unmapped":unmapped,"jobs_disabled":disabled},"jobs":results}
   137	state["authority"]="canonical"
   138	state["compatibility_map"]=str(ROOT/"compat-map.json")
   139	state["summary"]["compatibility_outputs"]=len(compat_map)
   140	atomic(ROOT/"state.json",json.dumps(state,ensure_ascii=False,indent=2,sort_keys=True)+chr(10))
===== FILE /usr/local/sbin/homelab-vm160-remote-backup =====
/usr/local/sbin/homelab-vm160-remote-backup: Bourne-Again shell script, ASCII text executable
     1	#!/usr/bin/env bash
     2	set -Eeuo pipefail
     3	export SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK
     4	VMID=160
     5	SOURCE=root@[PRIVATE_IP]
     6	TARGET=root@[PRIVATE_IP]
     7	KH=[SENSITIVE_PATH]
     8	PARENT=/mnt/staging/vzdump
     9	PREFIX=vm160-pve02-
    10	HEALTH=/var/lib/homelab-health/vm160-remote-backup.txt
    11	LOCK=/run/lock/homelab-vm160-remote-backup.lock
    12	SSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$KH")
    13	TARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker
    14	target_worker(){ ssh "${SSH[@]}" "$TARGET" bash -s -- "$@" < "$TARGET_WORKER"; }
    15	install -d -m 0755 "$(dirname "$HEALTH")" "$(dirname "$LOCK")"
    16	exec 9>"$LOCK"
    17	flock -n 9
    18	test "$(hostname -s)" = pve01
    19	test -x "$TARGET_WORKER"
    20	test "$(ssh "${SSH[@]}" "$SOURCE" "qm status $VMID" | awk '{print $2}')" = running
    21	SOURCE_ACTIVE_RC=0
    22	SOURCE_ACTIVE="$(ssh "${SSH[@]}" "$SOURCE" "pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'")" || SOURCE_ACTIVE_RC=$?
    23	test "$SOURCE_ACTIVE_RC" -eq 0 || test "$SOURCE_ACTIVE_RC" -eq 1
    24	test -z "$SOURCE_ACTIVE"
    25	TS="$(date -u +%Y%m%dT%H%M%SZ)"
    26	DIR="$PARENT/${PREFIX}${TS}"
    27	NAME="vzdump-qemu-${VMID}-${TS}.vma.zst"
    28	PART="$DIR/.${NAME}.partial"
    29	FINAL="$DIR/$NAME"
    30	LOG="$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)"
    31	cleanup(){
    32	  rc=$?
    33	  trap - EXIT
    34	  if test "$rc" -ne 0; then
    35	    ssh "${SSH[@]}" "$TARGET" "rm -rf -- '$DIR'" >/dev/null 2>&1 || true
    36	    tmp="${HEALTH}.tmp"
    37	    printf 'STATUS=FAIL\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nTARGET_DIR=%s\nRC=%s\n' "$TS" "$VMID" "$DIR" "$rc" > "$tmp"
    38	    chmod 0644 "$tmp"
    39	    mv "$tmp" "$HEALTH"
    40	  fi
    41	  rm -f "$LOG"
    42	  exit "$rc"
    43	}
    44	trap cleanup EXIT
    45	REMOTE_FREE="$(ssh "${SSH[@]}" "$TARGET" "df -B1 --output=avail '$PARENT' | awk 'NR==2{print \$1}'")"
    46	LATEST_SIZE="$(ssh "${SSH[@]}" "$TARGET" "find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\n' | sort -nr | head -n1")"
    47	LATEST_SIZE="${LATEST_SIZE:-0}"
    48	if test "$LATEST_SIZE" -gt 0; then
    49	  REQUIRED="$((LATEST_SIZE + LATEST_SIZE / 4 + 10737418240))"
    50	else
    51	  REQUIRED=161061273600
    52	fi
    53	test "$REMOTE_FREE" -ge "$REQUIRED"
    54	ssh "${SSH[@]}" "$TARGET" "install -d -m 0700 '$DIR'"
    55	ssh "${SSH[@]}" "$SOURCE" "vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7" 2>"$LOG" | ssh "${SSH[@]}" "$TARGET" "cat > '$PART'"
    56	target_worker finalize "$PARENT" "$PREFIX" "$PART" "$FINAL" "$DIR" "$NAME"
    57	ssh "${SSH[@]}" "$TARGET" "cat > '$DIR/vzdump.log'" < "$LOG"
    58	SIZE="$(ssh "${SSH[@]}" "$TARGET" "stat -c %s '$FINAL'")"
    59	SHA="$(ssh "${SSH[@]}" "$TARGET" "sha256sum '$FINAL' | awk '{print \$1}'")"
    60	target_worker promote "$PARENT" "$PREFIX" "$DIR" "$TS"
    61	tmp="${HEALTH}.tmp"
    62	printf 'STATUS=OK\nSERVICE=homelab-vm160-remote-backup\nTIMESTAMP_UTC=%s\nVMID=%s\nSOURCE_NODE=pve02\nTARGET_NODE=pve03\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\nREMOTE_RETENTION_GENERATIONS=1\n' "$TS" "$VMID" "$DIR" "$FINAL" "$SIZE" "$SHA" > "$tmp"
    63	chmod 0644 "$tmp"
    64	mv "$tmp" "$HEALTH"
    65	trap - EXIT
    66	rm -f "$LOG"
    67	printf 'VM160_REMOTE_BACKUP_STATUS=OK\nTARGET_DIR=%s\nARCHIVE=%s\nSIZE_BYTES=%s\nSHA256=%s\n' "$DIR" "$FINAL" "$SIZE" "$SHA"
===== FILE /usr/local/sbin/homelab-cluster-admin-pull =====
/usr/local/sbin/homelab-cluster-admin-pull: Bourne-Again shell script, ASCII text executable, with very long lines (424)
     1	#!/usr/bin/env bash
     2	set +e +u +o pipefail 2>/dev/null || true
     3	TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)
     4	ROOT=/var/lib/homelab-cluster-admin
     5	HEALTH_ROOT=/var/lib/homelab-health
     6	mkdir -p "$ROOT" "$HEALTH_ROOT"
     7	REMOTE=$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf "__OBS__\\n"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf "__FULL__\\n"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err)
     8	RC=$?
     9	OBS=$(printf "%s\n" "$REMOTE" | awk '/^__OBS__$/{f=1;next}/^__FULL__$/{f=0}f')
    10	FULL=$(printf "%s\n" "$REMOTE" | awk '/^__FULL__$/{f=1;next}f')
    11	OBS_BAD=0; FULL_BAD=0
    12	printf "%s\n" "$OBS" | grep -q "^STATUS=OK" || OBS_BAD=1
    13	printf "%s\n" "$FULL" | grep -q "^STATUS=OK" || FULL_BAD=1
    14	printf "%s\n" "$FULL" | grep -q "BAD=0" || FULL_BAD=1
    15	PROBE_OK=$(printf "%s\n" "$FULL" | sed -n 's/.*PROBE_OK=\([0-9][0-9]*\).*/\1/p' | head -n1)
    16	PROBE_BAD=$(printf "%s\n" "$FULL" | sed -n 's/.*PROBE_BAD=\([0-9][0-9]*\).*/\1/p' | head -n1)
    17	[ "${PROBE_BAD:-1}" = 0 ] || FULL_BAD=1
    18	BAD=$((OBS_BAD+FULL_BAD)); [ "$RC" -eq 0 ] || BAD=$((BAD+1))
    19	printf "%s\n" "$OBS" > "$ROOT/observer-remote-latest.txt"
    20	printf "%s\n" "$FULL" > "$ROOT/full-observer-central-latest.md"
    21	OBS_STATUS=$([ "$OBS_BAD" -eq 0 ] && [ "$RC" -eq 0 ] && echo OK || echo WARN)
    22	FULL_STATUS=$([ "$FULL_BAD" -eq 0 ] && [ "$RC" -eq 0 ] && echo OK || echo WARN)
    23	echo "STATUS=$OBS_STATUS TS=$TS TYPE=cluster-admin-central-pull BAD=$OBS_BAD VMID=180 IP=[PRIVATE_IP] SSH_RC=$RC OBSERVER_STATUS=$OBS_STATUS REPORT=$ROOT/observer-remote-latest.txt" > "$HEALTH_ROOT/cluster-admin-observer.txt"
    24	echo "STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-full-observer-central BAD=$FULL_BAD WARN=0 VMID=180 PULL_RC=$RC FULL_STATUS=$FULL_STATUS PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} REPORT=$ROOT/full-observer-central-latest.md" > "$HEALTH_ROOT/cluster-admin-full-observer.txt"
    25	echo "STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-restricted-probes BAD=$FULL_BAD VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer" > "$HEALTH_ROOT/cluster-admin-restricted-probes.txt"
    26	cat "$HEALTH_ROOT/cluster-admin-observer.txt"
    27	cat "$HEALTH_ROOT/cluster-admin-full-observer.txt"
    28	cat "$HEALTH_ROOT/cluster-admin-restricted-probes.txt"
    29	[ "$BAD" -eq 0 ]
CHANGES_MADE=NO

OUTPUT_END
CHAT_OUTPUT_END
