homelab-runtime-publisher
31 lines
19 KiB
JSON
31 lines
19 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"channel": "homelab-runtime",
|
|
"command_id": "BACKUP-FAILED-UNITS-SOURCE-217",
|
|
"status": "OK",
|
|
"rc": 0,
|
|
"host": "pve01",
|
|
"mode": "read-only",
|
|
"component": "cluster-audit",
|
|
"started_at_utc": "2026-07-22T15:35:58Z",
|
|
"finished_at_utc": "2026-07-22T15:35:58Z",
|
|
"reference_register_checked": true,
|
|
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
|
|
"error_register_checked": true,
|
|
"error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
|
|
"command_sha256": "2839e6de5b682a8f42432e3841c7b452be12e60f73cd670b0ac4412372ad4c86",
|
|
"duplicate_failed_command_blocked": false,
|
|
"block_reason": null,
|
|
"execution_started": true,
|
|
"changes_made": false,
|
|
"sanitized": true,
|
|
"secrets_included": false,
|
|
"private_addresses_included": false,
|
|
"raw_evidence_retained_locally": true,
|
|
"raw_evidence_sha256": "96e72a8cec4b5e5cde52a48e6816f4ce162311b07530ee398c57efc67f3480b1",
|
|
"sanitized_output_sha256": "dbe6bb8b0419312d9774755ff8668c435118bc102b49b0255667d04f06dda499",
|
|
"output_truncated_in_json": false,
|
|
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
|
|
"output": "PREVIOUS_COMMAND_ID=RUNTIME-PUBLISH-GAP-DIAG-216\n===== UNIT homelab-backup-matrix-refresh =====\n# /etc/systemd/system/homelab-backup-matrix-refresh.service\n[Unit]\nDescription=Generate unified homelab backup matrix state\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-backup-matrix-refresh\nTimeoutStartSec=5min\n===== UNIT homelab-vm160-remote-backup =====\n# /etc/systemd/system/homelab-vm160-remote-backup.service\n[Unit]\nDescription=VM160 full-image backup streamed from pve02 to pve03\nAfter=network-online.target pve-cluster.service\nWants=network-online.target\n\n[Service]\nType=oneshot\nTimeoutStartSec=6h\nExecStart=/usr/local/sbin/homelab-vm160-remote-backup\n===== UNIT homelab-cluster-admin-pull =====\n# /etc/systemd/system/homelab-cluster-admin-pull.service\n[Unit]\nDescription=Pull cluster-admin observer health into pve01\n\n[Service]\nType=oneshot\nExecStart=/usr/local/sbin/homelab-cluster-admin-pull\n===== FILE /usr/local/sbin/homelab-backup-matrix-refresh =====\n/usr/local/sbin/homelab-backup-matrix-refresh: Python script, ASCII text executable, with very long lines (592)\n 1\t#!/usr/bin/env python3\n 2\tfrom pathlib import Path\n 3\timport shutil\n 4\timport json,subprocess,os,time,sys,hashlib\n 5\tM=Path(\"/etc/homelab-backup-matrix/matrix.json\")\n 6\tROOT=Path(\"/var/lib/homelab-backup-matrix\")\n 7\tRENDER=ROOT/\"rendered\"\n 8\tCOMPAT=ROOT/\"compat\"\n 9\tdef cmd(args):\n 10\t p=subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,check=False)\n 11\t return p.stdout.strip() or \"unknown\"\n 12\tdef atomic(path,text,mode=0o640):\n 13\t path.parent.mkdir(parents=True,exist_ok=True)\n 14\t tmp=path.with_name(path.name+\".tmp\")\n 15\t tmp.write_text(text,encoding=\"utf-8\")\n 16\t os.chmod(tmp,mode)\n 17\t os.replace(tmp,path)\n 18\tdef parse_health(path):\n 19\t if path.suffix.lower()==\".json\":\n 20\t try:\n 21\t data=json.loads(path.read_text(encoding=\"utf-8\",errors=\"replace\"))\n 22\t return str(data.get(\"status\",data.get(\"STATUS\",data.get(\"result\",\"UNKNOWN\"))))\n 23\t except Exception:\n 24\t return \"ERROR\"\n 25\t data={}\n 26\t text=path.read_text(encoding=\"utf-8\",errors=\"replace\")\n 27\t for token in text.replace(chr(10),\" \").split():\n 28\t if \"=\" in token:\n 29\t key,value=token.split(\"=\",1)\n 30\t if key and key[0].isalpha():\n 31\t data[key]=value\n 32\t return str(data.get(\"STATUS\",data.get(\"status\",\"UNKNOWN\")))\n 33\tdef normalize(value):\n 34\t value=str(value).strip().upper()\n 35\t if value in {\"OK\",\"PASS\",\"PASSED\",\"SUCCESS\",\"HEALTHY\"}:\n 36\t return \"OK\"\n 37\t if value in {\"WARN\",\"WARNING\"}:\n 38\t return \"WARN\"\n 39\t if value in {\"FAIL\",\"FAILED\",\"ERROR\",\"CRITICAL\",\"BAD\"}:\n 40\t return \"FAIL\"\n 41\t return value or \"UNKNOWN\"\n 42\tmatrix=json.loads(M.read_text(encoding=\"utf-8\"))\n 43\tROOT.mkdir(parents=True,exist_ok=True)\n 44\tRENDER.mkdir(parents=True,exist_ok=True)\n 45\tCOMPAT.mkdir(parents=True,exist_ok=True)\n 46\tnow=int(time.time())\n 47\tnow_iso=time.strftime(\"%Y-%m-%dT%H:%M:%SZ\",time.gmtime(now))\n 48\tresults=[]\n 49\tcompat_map=[]\n 50\tfor job in matrix.get(\"jobs\",[]):\n 51\t original_sources=list(job.get(\"evidence_sources_original\",[]))\n 52\t evidence_inputs=list(job.get(\"evidence_inputs\",[]))\n 53\t if original_sources and len(original_sources)!=len(evidence_inputs):\n 54\t raise RuntimeError(\"evidence_mapping_length_mismatch:\"+str(job.get(\"name\",\"\")))\n 55\t for source_raw,destination_raw in zip(original_sources,evidence_inputs):\n 56\t source_path=Path(source_raw)\n 57\t destination_path=Path(destination_raw)\n 58\t if not source_path.is_file():\n 59\t continue\n 60\t head=source_path.read_text(encoding=\"utf-8\",errors=\"ignore\")[:4096]\n 61\t if \"TYPE=backup-matrix-generated\" in head or \"CANONICAL_STATE=/var/lib/homelab-backup-matrix/state.json\" in head:\n 62\t continue\n 63\t destination_path.parent.mkdir(parents=True,exist_ok=True)\n 64\t temporary=destination_path.with_name(destination_path.name+\".tmp-ingest\")\n 65\t shutil.copy2(source_path,temporary)\n 66\t os.replace(temporary,destination_path)\n 67\t jid=str(job.get(\"id\",\"\"))\n 68\t name=str(job.get(\"name\",jid))\n 69\t timer=str(job.get(\"timer\",\"\"))\n 70\t service=str(job.get(\"service\",\"\"))\n 71\t enabled=cmd([\"systemctl\",\"is-enabled\",timer]) if timer else \"missing\"\n 72\t active=cmd([\"systemctl\",\"is-active\",timer]) if timer else \"missing\"\n 73\t source_kind=str(job.get(\"source_kind\",\"health_file\"))\n 74\t best=None\n 75\t for raw in job.get(\"evidence_inputs\",job.get(\"legacy_health_inputs\",[])):\n 76\t path=Path(raw)\n 77\t if not path.is_file():\n 78\t continue\n 79\t mtime=int(path.stat().st_mtime)\n 80\t if best is None or mtime>best[0]:\n 81\t best=(mtime,str(path),parse_health(path))\n 82\t service_result=cmd([\"systemctl\",\"show\",service,\"-p\",\"Result\",\"--value\"]) if service else \"missing\"\n 83\t service_exec_main_status=cmd([\"systemctl\",\"show\",service,\"-p\",\"ExecMainStatus\",\"--value\"]) if service else \"missing\"\n 84\t evidence_level=\"artifact_or_health\"\n 85\t if best is None and source_kind==\"systemd\":\n 86\t stamp=Path(\"/var/lib/systemd/timers\")/(\"stamp-\"+timer)\n 87\t mtime=int(stamp.stat().st_mtime) if stamp.is_file() else 0\n 88\t source=\"systemd:\"+service\n 89\t source_status=\"OK\" if service_result==\"success\" and service_exec_main_status==\"0\" else \"FAIL\"\n 90\t evidence_level=\"execution_only\"\n 91\t else:\n 92\t source=best[1] if best else \"\"\n 93\t source_status=normalize(best[2]) if best else \"UNMAPPED\"\n 94\t mtime=best[0] if best else 0\n 95\t age=now-mtime if mtime else -1\n 96\t max_age=int(job.get(\"max_age_seconds\",172800))\n 97\t critical=bool(job.get(\"critical\",False))\n 98\t if not critical and enabled!=\"enabled\":\n 99\t final=\"DISABLED\"\n 100\t elif critical and (enabled!=\"enabled\" or active!=\"active\"):\n 101\t final=\"FAIL\"\n 102\t elif source_status==\"UNMAPPED\":\n 103\t final=\"UNMAPPED\"\n 104\t elif source_status!=\"OK\":\n 105\t final=\"FAIL\"\n 106\t elif age<0:\n 107\t final=\"MISSING\"\n 108\t elif age>max_age:\n 109\t final=\"STALE\"\n 110\t else:\n 111\t final=\"OK\"\n 112\t freshness=\"UNKNOWN\" if age<0 else (\"FRESH\" if age<=max_age else \"STALE\")\n 113\t source_mtime=time.strftime(\"%Y-%m-%dT%H:%M:%SZ\",time.gmtime(mtime)) if mtime else \"\"\n 114\t result={\"id\":jid,\"name\":name,\"category\":job.get(\"category\",\"unknown\"),\"critical\":critical,\"timer\":timer,\"service\":service,\"timer_enabled\":enabled,\"timer_active\":active,\"status\":final,\"source_kind\":source_kind,\"evidence_level\":evidence_level,\"service_result\":service_result,\"service_exec_main_status\":service_exec_main_status,\"source_health\":source,\"source_status\":source_status,\"source_mtime\":source_mtime,\"age_seconds\":age,\"max_age_seconds\":max_age,\"freshness\":freshness,\"generated_health_output\":job.get(\"generated_health_output\",\"\")}\n 115\t results.append(result)\n 116\t mode_value=matrix.get(\"mode\",\"shadow\"); lines=[f\"STATUS={final}\",f\"TS={now_iso}\",\"TYPE=backup-matrix-generated\",f\"JOB_ID={jid}\",f\"JOB_NAME={name}\",f\"MATRIX_MODE={mode_value}\",f\"TIMER={timer}\",f\"SERVICE={service}\",f\"TIMER_ENABLED={enabled}\",f\"TIMER_ACTIVE={active}\",f\"SOURCE_KIND={source_kind}\",f\"EVIDENCE_LEVEL={evidence_level}\",f\"SERVICE_RESULT={service_result}\",f\"SERVICE_EXEC_MAIN_STATUS={service_exec_main_status}\",f\"SOURCE_HEALTH={source}\",f\"SOURCE_STATUS={source_status}\",f\"SOURCE_MTIME_UTC={source_mtime}\",f\"AGE_SECONDS={age}\",f\"MAX_AGE_SECONDS={max_age}\",f\"FRESHNESS={freshness}\"]\n 117\t atomic(Path(job.get(\"generated_health_output\",str(RENDER/(name+\".txt\")))),chr(10).join(lines)+chr(10))\n 118\t for raw in job.get(\"compatibility_outputs\",[]):\n 119\t legacy=str(raw)\n 120\t digest=hashlib.sha256(legacy.encode(\"utf-8\")).hexdigest()[:12]\n 121\t base_name=Path(legacy).name or \"health\"\n 122\t safe=f\"{jid}--{base_name}--{digest}.txt\"\n 123\t compat_path=COMPAT/safe\n 124\t compat_lines=lines+[f\"LEGACY_SOURCE_PATH={legacy}\",f\"CANONICAL_JOB_ID={jid}\",f\"CANONICAL_STATE={ROOT}/state.json\"]\n 125\t atomic(compat_path,chr(10).join(compat_lines)+chr(10))\n 126\t atomic(Path(legacy),chr(10).join(compat_lines)+chr(10))\n 127\t compat_map.append({\"job_id\":jid,\"legacy_path\":legacy,\"compatibility_path\":str(compat_path)})\n 128\tbad=sum(1 for item in results if item[\"status\"] in {\"FAIL\",\"STALE\",\"MISSING\"})\n 129\tunmapped=sum(1 for item in results if item[\"status\"]==\"UNMAPPED\")\n 130\tdisabled=sum(1 for item in results if item[\"status\"]==\"DISABLED\")\n 131\tok=sum(1 for item in results if item[\"status\"]==\"OK\")\n 132\tmode=str(matrix.get(\"mode\",\"shadow\"))\n 133\toverall=\"OK\" if bad==0 and unmapped==0 else (\"WARN\" if mode==\"shadow\" else \"FAIL\")\n 134\tcompat_map=sorted(compat_map,key=lambda item:(item[\"legacy_path\"],item[\"job_id\"]))\n 135\tatomic(ROOT/\"compat-map.json\",json.dumps(compat_map,ensure_ascii=False,indent=2,sort_keys=True)+chr(10))\n 136\tstate={\"version\":1,\"generated_at\":now_iso,\"matrix_mode\":mode,\"matrix_sha256\":hashlib.sha256(M.read_bytes()).hexdigest(),\"summary\":{\"status\":overall,\"jobs_total\":len(results),\"jobs_ok\":ok,\"jobs_bad\":bad,\"jobs_unmapped\":unmapped,\"jobs_disabled\":disabled},\"jobs\":results}\n 137\tstate[\"authority\"]=\"canonical\"\n 138\tstate[\"compatibility_map\"]=str(ROOT/\"compat-map.json\")\n 139\tstate[\"summary\"][\"compatibility_outputs\"]=len(compat_map)\n 140\tatomic(ROOT/\"state.json\",json.dumps(state,ensure_ascii=False,indent=2,sort_keys=True)+chr(10))\n===== FILE /usr/local/sbin/homelab-vm160-remote-backup =====\n/usr/local/sbin/homelab-vm160-remote-backup: Bourne-Again shell script, ASCII text executable\n 1\t#!/usr/bin/env bash\n 2\tset -Eeuo pipefail\n 3\texport SYSTEMD_PAGER=cat PAGER=cat SYSTEMD_LESS=FRXMK\n 4\tVMID=160\n 5\tSOURCE=root@[PRIVATE_IP]\n 6\tTARGET=root@[PRIVATE_IP]\n 7\tKH=[SENSITIVE_PATH]\n 8\tPARENT=/mnt/staging/vzdump\n 9\tPREFIX=vm160-pve02-\n 10\tHEALTH=/var/lib/homelab-health/vm160-remote-backup.txt\n 11\tLOCK=/run/lock/homelab-vm160-remote-backup.lock\n 12\tSSH=(-o BatchMode=yes -o ConnectTimeout=12 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=\"$KH\")\n 13\tTARGET_WORKER=/usr/local/libexec/homelab-vm160-target-worker\n 14\ttarget_worker(){ ssh \"${SSH[@]}\" \"$TARGET\" bash -s -- \"$@\" < \"$TARGET_WORKER\"; }\n 15\tinstall -d -m 0755 \"$(dirname \"$HEALTH\")\" \"$(dirname \"$LOCK\")\"\n 16\texec 9>\"$LOCK\"\n 17\tflock -n 9\n 18\ttest \"$(hostname -s)\" = pve01\n 19\ttest -x \"$TARGET_WORKER\"\n 20\ttest \"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"qm status $VMID\" | awk '{print $2}')\" = running\n 21\tSOURCE_ACTIVE_RC=0\n 22\tSOURCE_ACTIVE=\"$(ssh \"${SSH[@]}\" \"$SOURCE\" \"pgrep -af '(^|/)(vzdump|qmrestore|pctrestore|vma|zstd)([[:space:]]|$)'\")\" || SOURCE_ACTIVE_RC=$?\n 23\ttest \"$SOURCE_ACTIVE_RC\" -eq 0 || test \"$SOURCE_ACTIVE_RC\" -eq 1\n 24\ttest -z \"$SOURCE_ACTIVE\"\n 25\tTS=\"$(date -u +%Y%m%dT%H%M%SZ)\"\n 26\tDIR=\"$PARENT/${PREFIX}${TS}\"\n 27\tNAME=\"vzdump-qemu-${VMID}-${TS}.vma.zst\"\n 28\tPART=\"$DIR/.${NAME}.partial\"\n 29\tFINAL=\"$DIR/$NAME\"\n 30\tLOG=\"$(mktemp /tmp/vm160-remote-backup.XXXXXX.log)\"\n 31\tcleanup(){\n 32\t rc=$?\n 33\t trap - EXIT\n 34\t if test \"$rc\" -ne 0; then\n 35\t ssh \"${SSH[@]}\" \"$TARGET\" \"rm -rf -- '$DIR'\" >/dev/null 2>&1 || true\n 36\t tmp=\"${HEALTH}.tmp\"\n 37\t printf 'STATUS=FAIL\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nTARGET_DIR=%s\\nRC=%s\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$rc\" > \"$tmp\"\n 38\t chmod 0644 \"$tmp\"\n 39\t mv \"$tmp\" \"$HEALTH\"\n 40\t fi\n 41\t rm -f \"$LOG\"\n 42\t exit \"$rc\"\n 43\t}\n 44\ttrap cleanup EXIT\n 45\tREMOTE_FREE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"df -B1 --output=avail '$PARENT' | awk 'NR==2{print \\$1}'\")\"\n 46\tLATEST_SIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"find '$PARENT' -mindepth 2 -maxdepth 2 -type f -path '$PARENT/${PREFIX}*/vzdump-qemu-${VMID}-*.vma.zst' -printf '%s\\n' | sort -nr | head -n1\")\"\n 47\tLATEST_SIZE=\"${LATEST_SIZE:-0}\"\n 48\tif test \"$LATEST_SIZE\" -gt 0; then\n 49\t REQUIRED=\"$((LATEST_SIZE + LATEST_SIZE / 4 + 10737418240))\"\n 50\telse\n 51\t REQUIRED=161061273600\n 52\tfi\n 53\ttest \"$REMOTE_FREE\" -ge \"$REQUIRED\"\n 54\tssh \"${SSH[@]}\" \"$TARGET\" \"install -d -m 0700 '$DIR'\"\n 55\tssh \"${SSH[@]}\" \"$SOURCE\" \"vzdump $VMID --stdout 1 --mode snapshot --compress zstd --ionice 7\" 2>\"$LOG\" | ssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$PART'\"\n 56\ttarget_worker finalize \"$PARENT\" \"$PREFIX\" \"$PART\" \"$FINAL\" \"$DIR\" \"$NAME\"\n 57\tssh \"${SSH[@]}\" \"$TARGET\" \"cat > '$DIR/vzdump.log'\" < \"$LOG\"\n 58\tSIZE=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"stat -c %s '$FINAL'\")\"\n 59\tSHA=\"$(ssh \"${SSH[@]}\" \"$TARGET\" \"sha256sum '$FINAL' | awk '{print \\$1}'\")\"\n 60\ttarget_worker promote \"$PARENT\" \"$PREFIX\" \"$DIR\" \"$TS\"\n 61\ttmp=\"${HEALTH}.tmp\"\n 62\tprintf 'STATUS=OK\\nSERVICE=homelab-vm160-remote-backup\\nTIMESTAMP_UTC=%s\\nVMID=%s\\nSOURCE_NODE=pve02\\nTARGET_NODE=pve03\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\nREMOTE_RETENTION_GENERATIONS=1\\n' \"$TS\" \"$VMID\" \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\" > \"$tmp\"\n 63\tchmod 0644 \"$tmp\"\n 64\tmv \"$tmp\" \"$HEALTH\"\n 65\ttrap - EXIT\n 66\trm -f \"$LOG\"\n 67\tprintf 'VM160_REMOTE_BACKUP_STATUS=OK\\nTARGET_DIR=%s\\nARCHIVE=%s\\nSIZE_BYTES=%s\\nSHA256=%s\\n' \"$DIR\" \"$FINAL\" \"$SIZE\" \"$SHA\"\n===== FILE /usr/local/sbin/homelab-cluster-admin-pull =====\n/usr/local/sbin/homelab-cluster-admin-pull: Bourne-Again shell script, ASCII text executable, with very long lines (424)\n 1\t#!/usr/bin/env bash\n 2\tset +e +u +o pipefail 2>/dev/null || true\n 3\tTS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\n 4\tROOT=/var/lib/homelab-cluster-admin\n 5\tHEALTH_ROOT=/var/lib/homelab-health\n 6\tmkdir -p \"$ROOT\" \"$HEALTH_ROOT\"\n 7\tREMOTE=$(ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@[PRIVATE_IP] 'sudo /opt/homelab-cluster-admin/cluster-observer.sh >/dev/null 2>&1 || true; sudo /opt/homelab-cluster-admin/full-observer.sh >/dev/null 2>&1 || true; printf \"__OBS__\\\\n\"; cat /var/lib/homelab-health/cluster-admin-observer.txt; printf \"__FULL__\\\\n\"; cat /var/lib/homelab-health/cluster-admin-full-observer.txt' 2>/tmp/homelab-cluster-admin-pull.err)\n 8\tRC=$?\n 9\tOBS=$(printf \"%s\\n\" \"$REMOTE\" | awk '/^__OBS__$/{f=1;next}/^__FULL__$/{f=0}f')\n 10\tFULL=$(printf \"%s\\n\" \"$REMOTE\" | awk '/^__FULL__$/{f=1;next}f')\n 11\tOBS_BAD=0; FULL_BAD=0\n 12\tprintf \"%s\\n\" \"$OBS\" | grep -q \"^STATUS=OK\" || OBS_BAD=1\n 13\tprintf \"%s\\n\" \"$FULL\" | grep -q \"^STATUS=OK\" || FULL_BAD=1\n 14\tprintf \"%s\\n\" \"$FULL\" | grep -q \"BAD=0\" || FULL_BAD=1\n 15\tPROBE_OK=$(printf \"%s\\n\" \"$FULL\" | sed -n 's/.*PROBE_OK=\\([0-9][0-9]*\\).*/\\1/p' | head -n1)\n 16\tPROBE_BAD=$(printf \"%s\\n\" \"$FULL\" | sed -n 's/.*PROBE_BAD=\\([0-9][0-9]*\\).*/\\1/p' | head -n1)\n 17\t[ \"${PROBE_BAD:-1}\" = 0 ] || FULL_BAD=1\n 18\tBAD=$((OBS_BAD+FULL_BAD)); [ \"$RC\" -eq 0 ] || BAD=$((BAD+1))\n 19\tprintf \"%s\\n\" \"$OBS\" > \"$ROOT/observer-remote-latest.txt\"\n 20\tprintf \"%s\\n\" \"$FULL\" > \"$ROOT/full-observer-central-latest.md\"\n 21\tOBS_STATUS=$([ \"$OBS_BAD\" -eq 0 ] && [ \"$RC\" -eq 0 ] && echo OK || echo WARN)\n 22\tFULL_STATUS=$([ \"$FULL_BAD\" -eq 0 ] && [ \"$RC\" -eq 0 ] && echo OK || echo WARN)\n 23\techo \"STATUS=$OBS_STATUS TS=$TS TYPE=cluster-admin-central-pull BAD=$OBS_BAD VMID=180 IP=[PRIVATE_IP] SSH_RC=$RC OBSERVER_STATUS=$OBS_STATUS REPORT=$ROOT/observer-remote-latest.txt\" > \"$HEALTH_ROOT/cluster-admin-observer.txt\"\n 24\techo \"STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-full-observer-central BAD=$FULL_BAD WARN=0 VMID=180 PULL_RC=$RC FULL_STATUS=$FULL_STATUS PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} REPORT=$ROOT/full-observer-central-latest.md\" > \"$HEALTH_ROOT/cluster-admin-full-observer.txt\"\n 25\techo \"STATUS=$FULL_STATUS TS=$TS TYPE=cluster-admin-restricted-probes BAD=$FULL_BAD VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=${PROBE_OK:-0} PROBE_BAD=${PROBE_BAD:-1} TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\" > \"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\"\n 26\tcat \"$HEALTH_ROOT/cluster-admin-observer.txt\"\n 27\tcat \"$HEALTH_ROOT/cluster-admin-full-observer.txt\"\n 28\tcat \"$HEALTH_ROOT/cluster-admin-restricted-probes.txt\"\n 29\t[ \"$BAD\" -eq 0 ]\nCHANGES_MADE=NO\n"
|
|
}
|