homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity
Files
5b4ad9dcc716dbd3c742d7df55b5b6bf47ad2c05
homelab-public-context/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z.txt
T

35 lines
85 KiB
Plaintext
Raw Blame History

CHAT_OUTPUT_BEGIN
COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z
STATUS=OK
RC=0
HOST=pve01
MODE=verify
COMPONENT=homelab-cluster-admin-update-visibility
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
COMMAND_SHA256=4845b4e99e60efaa533b2a551d4bf1773085ce96ef0e4ce9737b920907b0553f
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=b672b2fde18de662e73d8a98d4cc036895f9de2007679edc3c085392050e66e6
SANITIZED_OUTPUT_SHA256=c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8
OUTPUT_BEGIN
HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z","command_rc":0,"control_plane_error":null,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_corrected_atomic_transaction":true,"remote_application_candidate_count":40,"remote_hostname":"cluster-admin","remote_index_count":1,"remote_probe_privilege":"sudo-root","remote_reference_file_count":1,"remote_target_cardinality":1,"remote_target_host_sha256":"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69","remote_unit_count":16,"remote_webroot_config_count":0,"report_bytes":85056,"report_sha256":"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf","rollback_restored":false,"rollback_started":false,"status":"OK","v14_exact_result_identity_confirmed":true,"version":1,"webpanel_script_sha256":"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276"}
{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z","mutation_outcome":"NO_MUTATION","read_only":true,"remote_architecture":{"application_candidates":[{"bytes":133,"matches":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},{"bytes":149,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},{"bytes":154,"matches":[{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},{"bytes":151,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},{"bytes":231,"matches":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},{"bytes":154,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},{"bytes":138,"matches":[{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},{"bytes":127,"matches":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},{"bytes":616,"matches":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},{"bytes":1190,"matches":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},{"bytes":223,"matches":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},{"bytes":803,"matches":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},{"bytes":131,"matches":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},{"bytes":301,"matches":[{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},{"bytes":89,"matches":[{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f"},{"bytes":8444,"matches":[{"line":4,"text":"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')"},{"line":64,"text":" return f\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\n\""},{"line":82,"text":" old=<REDACTED>"},{"line":97,"text":" p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)"},{"line":99,"text":" STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))"}],"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a"},{"bytes":28599,"matches":[{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE =<REDACTED>"},{"line":14,"text":"COOKIE =<REDACTED>"},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":20,"text":" return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":73,"text":" return f\"\"\"<!doctype html><html lang=<REDACTED>"},{"line":152,"text":" return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":220,"text":" page = f\"\"\"<!doctype html><html lang=\"ru\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta http-equiv=\"refresh\" content=\"30\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\"post\" action=\"/logout\"><input type=\"hidden\" name=\"csrf\" value=\"{csrf}\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\"\"\""},{"line":243,"text":" \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":" \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":" \"cluster_admin_up 1\","},{"line":246,"text":" \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":" \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":" \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":" \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":" \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":" \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":" \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":" \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":" \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":" \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":" \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":" \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":" \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":" \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":" \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":" \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":" \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":" \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":267,"text":" lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6"},{"bytes":922,"matches":[{"line":2,"text":"first = getpass.getpass(\"Новый пароль Cluster Admin: \")"},{"line":12,"text":"directory = \"/etc/cluster-admin-incident-engine\""},{"line":16,"text":"account = pwd.getpwnam(\"clusteradmin\")"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3"},{"bytes":10640,"matches":[{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":36,"text":" data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":" connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":41,"text":" cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":50,"text":" if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":57,"text":" cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":108,"text":" line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe"},{"bytes":1857,"matches":[{"line":3,"text":" \"type\": \"cluster-admin-stage1-foundation-migration-plan\","},{"line":43,"text":" \"function\": \"cluster_admin_enrich_collector_run\","}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670"},{"bytes":1342,"matches":[{"line":3,"text":" \"collector_call\": \"SELECT cluster_admin_sync_foundation(%s)\","},{"line":15,"text":" \"function\": \"cluster_admin_sync_foundation\","},{"line":16,"text":" \"function_signature\": \"cluster_admin_sync_foundation(bigint)\","},{"line":26,"text":" \"type\": \"cluster-admin-stage2-migration-plan\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c"},{"bytes":2355,"matches":[{"line":51,"text":" \"type\": \"cluster-admin-stage2-dynamic-acceptance-contract\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588"},{"bytes":2832,"matches":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":" line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":" echo \"# Homelab cluster-admin observer\""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},{"bytes":3471,"matches":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":" echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":" echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},{"bytes":1433,"matches":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},{"bytes":848,"matches":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":10353,"matches":[{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a"},{"bytes":168,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify"}],"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729"},{"bytes":259,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741"},{"bytes":134,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517"},{"bytes":141,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6"},{"bytes":189,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941"},{"bytes":131,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0"}],"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":199,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680"},{"bytes":272,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec"},{"bytes":380,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"}],"euid":0,"gate":{"application_candidate_count":40,"ready_for_remote_transaction_design":true,"reference_file_count":1,"unit_count":16},"health_input_files":[{"bytes":380,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"},{"bytes":128,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac"},{"bytes":1535,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b"}],"hostname":"cluster-admin","indexes":[{"bytes":10353,"path":"/var/www/homelab-cluster-admin/index.html","root":"/var/www/homelab-cluster-admin","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a","title_lines":["<title>Homelab Cluster Admin</title>","<h1>Homelab Cluster Admin</h1>","<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>","<div class=<REDACTED>","<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>","<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>","<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer","pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"]}],"listeners":{"rc":0,"stderr":"","stdout":"State Recv-Q Send-Q Local Address:Port Peer Address:PortProcess \nLISTEN 0 5 0.0.0.0:8080 0.0.0.0:* users:((\"python3\",pid=439,fd=3)) \nLISTEN 0 2048 0.0.0.0:8081 0.0.0.0:* users:((\"python3\",pid=487,fd=6)) \nLISTEN 0 244 127.0.0.1:5432 0.0.0.0:* users:((\"postgres\",pid=469,fd=6)) \nLISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:((\"systemd-resolve\",pid=380,fd=18))\nLISTEN 0 4096 0.0.0.0:5355 0.0.0.0:* users:((\"systemd-resolve\",pid=380,fd=12))\nLISTEN 0 4096 127.0.0.54:53 0.0.0.0:* users:((\"systemd-resolve\",pid=380,fd=20))\nLISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:((\"sshd\",pid=457,fd=3)) \nLISTEN 0 244 [::1]:5432 [::]:* users:((\"postgres\",pid=469,fd=5)) \nLISTEN 0 4096 [::]:5355 [::]:* users:((\"systemd-resolve\",pid=380,fd=14))\nLISTEN 0 128 [::]:22 [::]:* users:((\"sshd\",pid=457,fd=4)) "},"matching_files":[{"bytes":133,"matches":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},{"bytes":149,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},{"bytes":154,"matches":[{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},{"bytes":151,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},{"bytes":231,"matches":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},{"bytes":154,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},{"bytes":138,"matches":[{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},{"bytes":127,"matches":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},{"bytes":616,"matches":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},{"bytes":1190,"matches":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},{"bytes":223,"matches":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},{"bytes":803,"matches":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},{"bytes":131,"matches":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},{"bytes":301,"matches":[{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},{"bytes":89,"matches":[{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f"},{"bytes":8444,"matches":[{"line":4,"text":"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')"},{"line":64,"text":" return f\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\n\""},{"line":82,"text":" old=<REDACTED>"},{"line":97,"text":" p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)"},{"line":99,"text":" STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))"}],"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a"},{"bytes":28599,"matches":[{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE =<REDACTED>"},{"line":14,"text":"COOKIE =<REDACTED>"},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":20,"text":" return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":73,"text":" return f\"\"\"<!doctype html><html lang=<REDACTED>"},{"line":152,"text":" return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":220,"text":" page = f\"\"\"<!doctype html><html lang=\"ru\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta http-equiv=\"refresh\" content=\"30\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\"post\" action=\"/logout\"><input type=\"hidden\" name=\"csrf\" value=\"{csrf}\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\"\"\""},{"line":243,"text":" \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":" \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":" \"cluster_admin_up 1\","},{"line":246,"text":" \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":" \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":" \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":" \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":" \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":" \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":" \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":" \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":" \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":" \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":" \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":" \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":" \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":" \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":" \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":" \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":" \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":" \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":267,"text":" lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6"},{"bytes":922,"matches":[{"line":2,"text":"first = getpass.getpass(\"Новый пароль Cluster Admin: \")"},{"line":12,"text":"directory = \"/etc/cluster-admin-incident-engine\""},{"line":16,"text":"account = pwd.getpwnam(\"clusteradmin\")"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3"},{"bytes":10640,"matches":[{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":36,"text":" data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":" connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":41,"text":" cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":50,"text":" if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":57,"text":" cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":108,"text":" line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe"},{"bytes":1857,"matches":[{"line":3,"text":" \"type\": \"cluster-admin-stage1-foundation-migration-plan\","},{"line":43,"text":" \"function\": \"cluster_admin_enrich_collector_run\","}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670"},{"bytes":1342,"matches":[{"line":3,"text":" \"collector_call\": \"SELECT cluster_admin_sync_foundation(%s)\","},{"line":15,"text":" \"function\": \"cluster_admin_sync_foundation\","},{"line":16,"text":" \"function_signature\": \"cluster_admin_sync_foundation(bigint)\","},{"line":26,"text":" \"type\": \"cluster-admin-stage2-migration-plan\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c"},{"bytes":2355,"matches":[{"line":51,"text":" \"type\": \"cluster-admin-stage2-dynamic-acceptance-contract\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588"},{"bytes":2832,"matches":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":" line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":" echo \"# Homelab cluster-admin observer\""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},{"bytes":3471,"matches":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":" echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":" echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},{"bytes":1433,"matches":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},{"bytes":848,"matches":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":10353,"matches":[{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a"},{"bytes":168,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify"}],"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729"},{"bytes":259,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741"},{"bytes":134,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517"},{"bytes":141,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6"},{"bytes":189,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941"},{"bytes":131,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0"}],"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":199,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680"},{"bytes":272,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec"},{"bytes":380,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":1535,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b"},{"bytes":128,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac"},{"bytes":217,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"}],"process_lines":[" 439 1 root root python3 /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"," 487 1 cluster+ cluster+ python3 /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1"],"proxies":[],"pve01_pushed_reference_files":[{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"}],"scan":{"directories_scanned":68,"index_count":1,"match_count":42},"units":[{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","Group":"clusteradmin","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","Group":"","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","Group":"root","LoadState":"loaded","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","User":"root","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}],"webroots":[]},"remote_target":{"evidence":[{"line":21,"target_count":1,"text":"scp -q -o BatchMode=yes -o ConnectTimeout=8 \"$TMP\"/*.txt debian@<PRIVATE_IPV4>:/tmp/"},{"line":24,"target_count":1,"text":"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \"$f\" && sudo install -m 0644 \"$f\" /var/lib/homelab-health/pve01-pushed/; done'"}],"target_cardinality":1,"target_host_is_private_ipv4":true,"target_host_sha256":"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69","target_user":"debian"},"rollback_restored":false,"rollback_started":false,"schema":15,"status":"OK","transaction_gate":{"application_architecture_found":true,"next_required_action":"build one corrected local+remote atomic transaction","pushed_health_reference_found":true,"ready_for_corrected_atomic_transaction":true,"remote_probe_root":true},"transport":{"attempts":[{"privilege":"sudo-root","rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":82754,"stdout_sha256":"603f5eb85c971d0bc029287cf91d9d0239b132dd409df5bbf8c5f47a7bb32738"}],"privilege":"sudo-root","success":true},"v14_exact_result":{"changes_made":false,"control_plane_error":null,"identity_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":10768,"output_sha256":"cc30d7f4e6be0ab5e78545d4537a39178b59e9f1a79227efaec134b78f6fc233","rollback_restored":null,"rollback_started":false,"runner_document_rc":0,"runner_json_sha256":"c932bbad4dd7f176e29621f3c549e2571baf7f84500ef2c5408a04c6a84083ff","runner_rc":0,"runner_status":"OK","runner_text_sha256":"70796c2456365c9224dd6ff02897d29412349eb1a5fea03c89ae9212217a03f8","runner_wrapper_matches_document":true,"state_document_sha256":"f3db8a869123d1266f906fdd02ca18a890141c2bd63e2bc57a3a0e7c5962f319","state_path_sha256":"2c050c8423a48aab8f448c94123e67f132381be3232e1d10274f4e87e1013b16","state_status":"OK"}}
OUTPUT_END
CHAT_OUTPUT_END
View Git Blame Copy Permalink