diff --git a/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z.json b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z.json
new file mode 100644
index 0000000..119b1d1
--- /dev/null
+++ b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z.json
@@ -0,0 +1,38 @@
+{
+  "schema_version": 1,
+  "channel": "homelab-runtime",
+  "command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z",
+  "status": "OK",
+  "rc": 0,
+  "host": "pve01",
+  "mode": "verify",
+  "component": "homelab-cluster-admin-update-visibility",
+  "started_at_utc": "2026-08-05T12:55:27Z",
+  "finished_at_utc": "2026-08-05T12:55:29Z",
+  "reference_register_checked": true,
+  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
+  "error_register_checked": true,
+  "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
+  "command_sha256": "4845b4e99e60efaa533b2a551d4bf1773085ce96ef0e4ce9737b920907b0553f",
+  "duplicate_failed_command_blocked": false,
+  "block_reason": null,
+  "execution_started": true,
+  "change_declared": false,
+  "result_contract_valid": true,
+  "result_contract_status": null,
+  "result_contract_error": null,
+  "command_rc": 0,
+  "changes_made": false,
+  "rollback_started": false,
+  "rollback_restored": null,
+  "mutation_outcome": "NO_MUTATION",
+  "sanitized": true,
+  "secrets_included": false,
+  "private_addresses_included": false,
+  "raw_evidence_retained_locally": true,
+  "raw_evidence_sha256": "b672b2fde18de662e73d8a98d4cc036895f9de2007679edc3c085392050e66e6",
+  "sanitized_output_sha256": "c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8",
+  "output_truncated_in_json": false,
+  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
+  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z\",\"command_rc\":0,\"control_plane_error\":null,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_corrected_atomic_transaction\":true,\"remote_application_candidate_count\":40,\"remote_hostname\":\"cluster-admin\",\"remote_index_count\":1,\"remote_probe_privilege\":\"sudo-root\",\"remote_reference_file_count\":1,\"remote_target_cardinality\":1,\"remote_target_host_sha256\":\"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69\",\"remote_unit_count\":16,\"remote_webroot_config_count\":0,\"report_bytes\":85056,\"report_sha256\":\"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"v14_exact_result_identity_confirmed\":true,\"version\":1,\"webpanel_script_sha256\":\"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276\"}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z\",\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"remote_architecture\":{\"application_candidates\":[{\"bytes\":133,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},{\"bytes\":149,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},{\"bytes\":151,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},{\"bytes\":231,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},{\"bytes\":138,\"matches\":[{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},{\"bytes\":127,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},{\"bytes\":616,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},{\"bytes\":1190,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},{\"bytes\":223,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},{\"bytes\":803,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},{\"bytes\":131,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},{\"bytes\":301,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},{\"bytes\":89,\"matches\":[{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\"},{\"bytes\":8444,\"matches\":[{\"line\":4,\"text\":\"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')\"},{\"line\":64,\"text\":\" return f\\\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\\\n\\\"\"},{\"line\":82,\"text\":\" old=<REDACTED>\"},{\"line\":97,\"text\":\"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)\"},{\"line\":99,\"text\":\"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\"},{\"bytes\":28599,\"matches\":[{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =<REDACTED>\"},{\"line\":14,\"text\":\"COOKIE =<REDACTED>\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":20,\"text\":\"    return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":73,\"text\":\"    return f\\\"\\\"\\\"<!doctype html><html lang=<REDACTED>\"},{\"line\":152,\"text\":\"    return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":220,\"text\":\"    page = f\\\"\\\"\\\"<!doctype html><html lang=\\\"ru\\\"><head><meta charset=\\\"utf-8\\\"><meta name=\\\"viewport\\\" content=\\\"width=device-width,initial-scale=1\\\"><meta http-equiv=\\\"refresh\\\" content=\\\"30\\\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\\\"post\\\" action=\\\"/logout\\\"><input type=\\\"hidden\\\" name=\\\"csrf\\\" value=\\\"{csrf}\\\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\\\"\\\"\\\"\"},{\"line\":243,\"text\":\"        \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\"        \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\"        \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\"        \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\"        \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\"        \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\"        \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\"        \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\"        \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\"        \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\"        \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\"        \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\"        \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\"        \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\"        \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\"        \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\"        \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\"        \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\"        \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\"        \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\"        \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":267,\"text\":\"        lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\"},{\"bytes\":922,\"matches\":[{\"line\":2,\"text\":\"first = getpass.getpass(\\\"Новый пароль Cluster Admin: \\\")\"},{\"line\":12,\"text\":\"directory = \\\"/etc/cluster-admin-incident-engine\\\"\"},{\"line\":16,\"text\":\"account = pwd.getpwnam(\\\"clusteradmin\\\")\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\"},{\"bytes\":10640,\"matches\":[{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":36,\"text\":\"    data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\"    connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":41,\"text\":\"        cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":50,\"text\":\"            if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":57,\"text\":\"        cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":108,\"text\":\"    line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\"},{\"bytes\":1857,\"matches\":[{\"line\":3,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage1-foundation-migration-plan\\\",\"},{\"line\":43,\"text\":\"    \\\"function\\\": \\\"cluster_admin_enrich_collector_run\\\",\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\"},{\"bytes\":1342,\"matches\":[{\"line\":3,\"text\":\"  \\\"collector_call\\\": \\\"SELECT cluster_admin_sync_foundation(%s)\\\",\"},{\"line\":15,\"text\":\"  \\\"function\\\": \\\"cluster_admin_sync_foundation\\\",\"},{\"line\":16,\"text\":\"  \\\"function_signature\\\": \\\"cluster_admin_sync_foundation(bigint)\\\",\"},{\"line\":26,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-migration-plan\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\"},{\"bytes\":2355,\"matches\":[{\"line\":51,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-dynamic-acceptance-contract\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\"},{\"bytes\":2832,\"matches\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\"  line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\"  echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},{\"bytes\":3471,\"matches\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\"  echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\"  echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},{\"bytes\":1433,\"matches\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},{\"bytes\":848,\"matches\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":10353,\"matches\":[{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\"},{\"bytes\":168,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729\"},{\"bytes\":259,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741\"},{\"bytes\":134,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517\"},{\"bytes\":141,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\"},{\"bytes\":189,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\"},{\"bytes\":131,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":199,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\"},{\"bytes\":272,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec\"},{\"bytes\":380,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"}],\"euid\":0,\"gate\":{\"application_candidate_count\":40,\"ready_for_remote_transaction_design\":true,\"reference_file_count\":1,\"unit_count\":16},\"health_input_files\":[{\"bytes\":380,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"},{\"bytes\":128,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\"},{\"bytes\":1535,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\"}],\"hostname\":\"cluster-admin\",\"indexes\":[{\"bytes\":10353,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"root\":\"/var/www/homelab-cluster-admin\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\",\"title_lines\":[\"<title>Homelab Cluster Admin</title>\",\"<h1>Homelab Cluster Admin</h1>\",\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\",\"<div class=<REDACTED>\",\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\",\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"]}],\"listeners\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"State  Recv-Q Send-Q Local Address:Port Peer Address:PortProcess                                   \\nLISTEN 0      5            0.0.0.0:8080      0.0.0.0:*    users:((\\\"python3\\\",pid=439,fd=3))         \\nLISTEN 0      2048         0.0.0.0:8081      0.0.0.0:*    users:((\\\"python3\\\",pid=487,fd=6))         \\nLISTEN 0      244        127.0.0.1:5432      0.0.0.0:*    users:((\\\"postgres\\\",pid=469,fd=6))        \\nLISTEN 0      4096   127.0.0.53%lo:53        0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=18))\\nLISTEN 0      4096         0.0.0.0:5355      0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=12))\\nLISTEN 0      4096      127.0.0.54:53        0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=20))\\nLISTEN 0      128          0.0.0.0:22        0.0.0.0:*    users:((\\\"sshd\\\",pid=457,fd=3))            \\nLISTEN 0      244            [::1]:5432         [::]:*    users:((\\\"postgres\\\",pid=469,fd=5))        \\nLISTEN 0      4096            [::]:5355         [::]:*    users:((\\\"systemd-resolve\\\",pid=380,fd=14))\\nLISTEN 0      128             [::]:22           [::]:*    users:((\\\"sshd\\\",pid=457,fd=4))            \"},\"matching_files\":[{\"bytes\":133,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},{\"bytes\":149,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},{\"bytes\":151,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},{\"bytes\":231,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},{\"bytes\":138,\"matches\":[{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},{\"bytes\":127,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},{\"bytes\":616,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},{\"bytes\":1190,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},{\"bytes\":223,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},{\"bytes\":803,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},{\"bytes\":131,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},{\"bytes\":301,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},{\"bytes\":89,\"matches\":[{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\"},{\"bytes\":8444,\"matches\":[{\"line\":4,\"text\":\"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')\"},{\"line\":64,\"text\":\" return f\\\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\\\n\\\"\"},{\"line\":82,\"text\":\" old=<REDACTED>\"},{\"line\":97,\"text\":\"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)\"},{\"line\":99,\"text\":\"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\"},{\"bytes\":28599,\"matches\":[{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =<REDACTED>\"},{\"line\":14,\"text\":\"COOKIE =<REDACTED>\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":20,\"text\":\"    return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":73,\"text\":\"    return f\\\"\\\"\\\"<!doctype html><html lang=<REDACTED>\"},{\"line\":152,\"text\":\"    return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":220,\"text\":\"    page = f\\\"\\\"\\\"<!doctype html><html lang=\\\"ru\\\"><head><meta charset=\\\"utf-8\\\"><meta name=\\\"viewport\\\" content=\\\"width=device-width,initial-scale=1\\\"><meta http-equiv=\\\"refresh\\\" content=\\\"30\\\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\\\"post\\\" action=\\\"/logout\\\"><input type=\\\"hidden\\\" name=\\\"csrf\\\" value=\\\"{csrf}\\\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\\\"\\\"\\\"\"},{\"line\":243,\"text\":\"        \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\"        \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\"        \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\"        \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\"        \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\"        \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\"        \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\"        \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\"        \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\"        \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\"        \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\"        \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\"        \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\"        \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\"        \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\"        \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\"        \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\"        \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\"        \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\"        \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\"        \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":267,\"text\":\"        lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\"},{\"bytes\":922,\"matches\":[{\"line\":2,\"text\":\"first = getpass.getpass(\\\"Новый пароль Cluster Admin: \\\")\"},{\"line\":12,\"text\":\"directory = \\\"/etc/cluster-admin-incident-engine\\\"\"},{\"line\":16,\"text\":\"account = pwd.getpwnam(\\\"clusteradmin\\\")\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\"},{\"bytes\":10640,\"matches\":[{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":36,\"text\":\"    data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\"    connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":41,\"text\":\"        cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":50,\"text\":\"            if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":57,\"text\":\"        cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":108,\"text\":\"    line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\"},{\"bytes\":1857,\"matches\":[{\"line\":3,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage1-foundation-migration-plan\\\",\"},{\"line\":43,\"text\":\"    \\\"function\\\": \\\"cluster_admin_enrich_collector_run\\\",\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\"},{\"bytes\":1342,\"matches\":[{\"line\":3,\"text\":\"  \\\"collector_call\\\": \\\"SELECT cluster_admin_sync_foundation(%s)\\\",\"},{\"line\":15,\"text\":\"  \\\"function\\\": \\\"cluster_admin_sync_foundation\\\",\"},{\"line\":16,\"text\":\"  \\\"function_signature\\\": \\\"cluster_admin_sync_foundation(bigint)\\\",\"},{\"line\":26,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-migration-plan\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\"},{\"bytes\":2355,\"matches\":[{\"line\":51,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-dynamic-acceptance-contract\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\"},{\"bytes\":2832,\"matches\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\"  line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\"  echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},{\"bytes\":3471,\"matches\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\"  echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\"  echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},{\"bytes\":1433,\"matches\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},{\"bytes\":848,\"matches\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":10353,\"matches\":[{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\"},{\"bytes\":168,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729\"},{\"bytes\":259,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741\"},{\"bytes\":134,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517\"},{\"bytes\":141,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\"},{\"bytes\":189,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\"},{\"bytes\":131,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":199,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\"},{\"bytes\":272,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec\"},{\"bytes\":380,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":1535,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\"},{\"bytes\":128,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\"},{\"bytes\":217,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"}],\"process_lines\":[\"    439       1 root     root     python3         /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\",\"    487       1 cluster+ cluster+ python3         /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\"],\"proxies\":[],\"pve01_pushed_reference_files\":[{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"}],\"scan\":{\"directories_scanned\":68,\"index_count\":1,\"match_count\":42},\"units\":[{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"Group\":\"root\",\"LoadState\":\"loaded\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"root\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}],\"webroots\":[]},\"remote_target\":{\"evidence\":[{\"line\":21,\"target_count\":1,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$TMP\\\"/*.txt debian@<PRIVATE_IPV4>:/tmp/\"},{\"line\":24,\"target_count\":1,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \\\"$f\\\" && sudo install -m 0644 \\\"$f\\\" /var/lib/homelab-health/pve01-pushed/; done'\"}],\"target_cardinality\":1,\"target_host_is_private_ipv4\":true,\"target_host_sha256\":\"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69\",\"target_user\":\"debian\"},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":15,\"status\":\"OK\",\"transaction_gate\":{\"application_architecture_found\":true,\"next_required_action\":\"build one corrected local+remote atomic transaction\",\"pushed_health_reference_found\":true,\"ready_for_corrected_atomic_transaction\":true,\"remote_probe_root\":true},\"transport\":{\"attempts\":[{\"privilege\":\"sudo-root\",\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":82754,\"stdout_sha256\":\"603f5eb85c971d0bc029287cf91d9d0239b132dd409df5bbf8c5f47a7bb32738\"}],\"privilege\":\"sudo-root\",\"success\":true},\"v14_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":10768,\"output_sha256\":\"cc30d7f4e6be0ab5e78545d4537a39178b59e9f1a79227efaec134b78f6fc233\",\"rollback_restored\":null,\"rollback_started\":false,\"runner_document_rc\":0,\"runner_json_sha256\":\"c932bbad4dd7f176e29621f3c549e2571baf7f84500ef2c5408a04c6a84083ff\",\"runner_rc\":0,\"runner_status\":\"OK\",\"runner_text_sha256\":\"70796c2456365c9224dd6ff02897d29412349eb1a5fea03c89ae9212217a03f8\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"f3db8a869123d1266f906fdd02ca18a890141c2bd63e2bc57a3a0e7c5962f319\",\"state_path_sha256\":\"2c050c8423a48aab8f448c94123e67f132381be3232e1d10274f4e87e1013b16\",\"state_status\":\"OK\"}}\n"
+}
diff --git a/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z.txt b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z.txt
new file mode 100644
index 0000000..04c4bc8
--- /dev/null
+++ b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z.txt
@@ -0,0 +1,34 @@
+CHAT_OUTPUT_BEGIN
+COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z
+STATUS=OK
+RC=0
+HOST=pve01
+MODE=verify
+COMPONENT=homelab-cluster-admin-update-visibility
+REFERENCE_REGISTER_CHECK=OK
+REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
+ERROR_REGISTER_CHECK=OK
+ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
+COMMAND_SHA256=4845b4e99e60efaa533b2a551d4bf1773085ce96ef0e4ce9737b920907b0553f
+DUPLICATE_FAILED_COMMAND_BLOCKED=false
+EXECUTION_STARTED=true
+CHANGE_DECLARED=false
+RESULT_CONTRACT_VALID=true
+RESULT_CONTRACT_STATUS=NOT_APPLICABLE
+RESULT_CONTRACT_ERROR=NONE
+COMMAND_RC=0
+CHANGES_MADE=false
+ROLLBACK_STARTED=false
+ROLLBACK_RESTORED=null
+MUTATION_OUTCOME=NO_MUTATION
+SANITIZED=yes
+SECRETS_INCLUDED=no
+PRIVATE_ADDRESSES_INCLUDED=no
+RAW_EVIDENCE_SHA256=b672b2fde18de662e73d8a98d4cc036895f9de2007679edc3c085392050e66e6
+SANITIZED_OUTPUT_SHA256=c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8
+OUTPUT_BEGIN
+HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z","command_rc":0,"control_plane_error":null,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_corrected_atomic_transaction":true,"remote_application_candidate_count":40,"remote_hostname":"cluster-admin","remote_index_count":1,"remote_probe_privilege":"sudo-root","remote_reference_file_count":1,"remote_target_cardinality":1,"remote_target_host_sha256":"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69","remote_unit_count":16,"remote_webroot_config_count":0,"report_bytes":85056,"report_sha256":"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf","rollback_restored":false,"rollback_started":false,"status":"OK","v14_exact_result_identity_confirmed":true,"version":1,"webpanel_script_sha256":"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276"}
+{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z","mutation_outcome":"NO_MUTATION","read_only":true,"remote_architecture":{"application_candidates":[{"bytes":133,"matches":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},{"bytes":149,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},{"bytes":154,"matches":[{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},{"bytes":151,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},{"bytes":231,"matches":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},{"bytes":154,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},{"bytes":138,"matches":[{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},{"bytes":127,"matches":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},{"bytes":616,"matches":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},{"bytes":1190,"matches":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},{"bytes":223,"matches":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},{"bytes":803,"matches":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},{"bytes":131,"matches":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},{"bytes":301,"matches":[{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},{"bytes":89,"matches":[{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f"},{"bytes":8444,"matches":[{"line":4,"text":"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')"},{"line":64,"text":" return f\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\n\""},{"line":82,"text":" old=<REDACTED>"},{"line":97,"text":"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)"},{"line":99,"text":"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))"}],"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a"},{"bytes":28599,"matches":[{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE =<REDACTED>"},{"line":14,"text":"COOKIE =<REDACTED>"},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":20,"text":"    return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":73,"text":"    return f\"\"\"<!doctype html><html lang=<REDACTED>"},{"line":152,"text":"    return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":220,"text":"    page = f\"\"\"<!doctype html><html lang=\"ru\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta http-equiv=\"refresh\" content=\"30\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\"post\" action=\"/logout\"><input type=\"hidden\" name=\"csrf\" value=\"{csrf}\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\"\"\""},{"line":243,"text":"        \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":"        \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":"        \"cluster_admin_up 1\","},{"line":246,"text":"        \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":"        \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":"        \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":"        \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":"        \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":"        \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":"        \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":"        \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":"        \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":"        \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":"        \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":"        \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":"        \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":"        \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":"        \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":"        \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":"        \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":"        \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":267,"text":"        lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6"},{"bytes":922,"matches":[{"line":2,"text":"first = getpass.getpass(\"Новый пароль Cluster Admin: \")"},{"line":12,"text":"directory = \"/etc/cluster-admin-incident-engine\""},{"line":16,"text":"account = pwd.getpwnam(\"clusteradmin\")"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3"},{"bytes":10640,"matches":[{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":36,"text":"    data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":"    connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":41,"text":"        cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":50,"text":"            if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":57,"text":"        cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":108,"text":"    line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe"},{"bytes":1857,"matches":[{"line":3,"text":"  \"type\": \"cluster-admin-stage1-foundation-migration-plan\","},{"line":43,"text":"    \"function\": \"cluster_admin_enrich_collector_run\","}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670"},{"bytes":1342,"matches":[{"line":3,"text":"  \"collector_call\": \"SELECT cluster_admin_sync_foundation(%s)\","},{"line":15,"text":"  \"function\": \"cluster_admin_sync_foundation\","},{"line":16,"text":"  \"function_signature\": \"cluster_admin_sync_foundation(bigint)\","},{"line":26,"text":"  \"type\": \"cluster-admin-stage2-migration-plan\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c"},{"bytes":2355,"matches":[{"line":51,"text":"  \"type\": \"cluster-admin-stage2-dynamic-acceptance-contract\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588"},{"bytes":2832,"matches":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},{"bytes":3471,"matches":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},{"bytes":1433,"matches":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},{"bytes":848,"matches":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":10353,"matches":[{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a"},{"bytes":168,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify"}],"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729"},{"bytes":259,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741"},{"bytes":134,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517"},{"bytes":141,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6"},{"bytes":189,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941"},{"bytes":131,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0"}],"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":199,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680"},{"bytes":272,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec"},{"bytes":380,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"}],"euid":0,"gate":{"application_candidate_count":40,"ready_for_remote_transaction_design":true,"reference_file_count":1,"unit_count":16},"health_input_files":[{"bytes":380,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"},{"bytes":128,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac"},{"bytes":1535,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b"}],"hostname":"cluster-admin","indexes":[{"bytes":10353,"path":"/var/www/homelab-cluster-admin/index.html","root":"/var/www/homelab-cluster-admin","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a","title_lines":["<title>Homelab Cluster Admin</title>","<h1>Homelab Cluster Admin</h1>","<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>","<div class=<REDACTED>","<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>","<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>","<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer","pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"]}],"listeners":{"rc":0,"stderr":"","stdout":"State  Recv-Q Send-Q Local Address:Port Peer Address:PortProcess                                   \nLISTEN 0      5            0.0.0.0:8080      0.0.0.0:*    users:((\"python3\",pid=439,fd=3))         \nLISTEN 0      2048         0.0.0.0:8081      0.0.0.0:*    users:((\"python3\",pid=487,fd=6))         \nLISTEN 0      244        127.0.0.1:5432      0.0.0.0:*    users:((\"postgres\",pid=469,fd=6))        \nLISTEN 0      4096   127.0.0.53%lo:53        0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=18))\nLISTEN 0      4096         0.0.0.0:5355      0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=12))\nLISTEN 0      4096      127.0.0.54:53        0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=20))\nLISTEN 0      128          0.0.0.0:22        0.0.0.0:*    users:((\"sshd\",pid=457,fd=3))            \nLISTEN 0      244            [::1]:5432         [::]:*    users:((\"postgres\",pid=469,fd=5))        \nLISTEN 0      4096            [::]:5355         [::]:*    users:((\"systemd-resolve\",pid=380,fd=14))\nLISTEN 0      128             [::]:22           [::]:*    users:((\"sshd\",pid=457,fd=4))            "},"matching_files":[{"bytes":133,"matches":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},{"bytes":149,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},{"bytes":154,"matches":[{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},{"bytes":151,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},{"bytes":231,"matches":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},{"bytes":154,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},{"bytes":138,"matches":[{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},{"bytes":127,"matches":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},{"bytes":616,"matches":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},{"bytes":1190,"matches":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},{"bytes":223,"matches":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},{"bytes":803,"matches":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},{"bytes":131,"matches":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},{"bytes":301,"matches":[{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},{"bytes":89,"matches":[{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f"},{"bytes":8444,"matches":[{"line":4,"text":"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')"},{"line":64,"text":" return f\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\n\""},{"line":82,"text":" old=<REDACTED>"},{"line":97,"text":"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)"},{"line":99,"text":"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))"}],"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a"},{"bytes":28599,"matches":[{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE =<REDACTED>"},{"line":14,"text":"COOKIE =<REDACTED>"},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":20,"text":"    return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":73,"text":"    return f\"\"\"<!doctype html><html lang=<REDACTED>"},{"line":152,"text":"    return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":220,"text":"    page = f\"\"\"<!doctype html><html lang=\"ru\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta http-equiv=\"refresh\" content=\"30\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\"post\" action=\"/logout\"><input type=\"hidden\" name=\"csrf\" value=\"{csrf}\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\"\"\""},{"line":243,"text":"        \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":"        \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":"        \"cluster_admin_up 1\","},{"line":246,"text":"        \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":"        \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":"        \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":"        \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":"        \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":"        \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":"        \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":"        \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":"        \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":"        \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":"        \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":"        \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":"        \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":"        \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":"        \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":"        \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":"        \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":"        \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":267,"text":"        lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6"},{"bytes":922,"matches":[{"line":2,"text":"first = getpass.getpass(\"Новый пароль Cluster Admin: \")"},{"line":12,"text":"directory = \"/etc/cluster-admin-incident-engine\""},{"line":16,"text":"account = pwd.getpwnam(\"clusteradmin\")"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3"},{"bytes":10640,"matches":[{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":36,"text":"    data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":"    connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":41,"text":"        cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":50,"text":"            if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":57,"text":"        cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":108,"text":"    line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe"},{"bytes":1857,"matches":[{"line":3,"text":"  \"type\": \"cluster-admin-stage1-foundation-migration-plan\","},{"line":43,"text":"    \"function\": \"cluster_admin_enrich_collector_run\","}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670"},{"bytes":1342,"matches":[{"line":3,"text":"  \"collector_call\": \"SELECT cluster_admin_sync_foundation(%s)\","},{"line":15,"text":"  \"function\": \"cluster_admin_sync_foundation\","},{"line":16,"text":"  \"function_signature\": \"cluster_admin_sync_foundation(bigint)\","},{"line":26,"text":"  \"type\": \"cluster-admin-stage2-migration-plan\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c"},{"bytes":2355,"matches":[{"line":51,"text":"  \"type\": \"cluster-admin-stage2-dynamic-acceptance-contract\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588"},{"bytes":2832,"matches":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},{"bytes":3471,"matches":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},{"bytes":1433,"matches":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},{"bytes":848,"matches":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":10353,"matches":[{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a"},{"bytes":168,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify"}],"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729"},{"bytes":259,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741"},{"bytes":134,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517"},{"bytes":141,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6"},{"bytes":189,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941"},{"bytes":131,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0"}],"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":199,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680"},{"bytes":272,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec"},{"bytes":380,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":1535,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b"},{"bytes":128,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac"},{"bytes":217,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"}],"process_lines":["    439       1 root     root     python3         /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin","    487       1 cluster+ cluster+ python3         /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1"],"proxies":[],"pve01_pushed_reference_files":[{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"}],"scan":{"directories_scanned":68,"index_count":1,"match_count":42},"units":[{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","Group":"clusteradmin","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","Group":"","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","Group":"root","LoadState":"loaded","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","User":"root","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}],"webroots":[]},"remote_target":{"evidence":[{"line":21,"target_count":1,"text":"scp -q -o BatchMode=yes -o ConnectTimeout=8 \"$TMP\"/*.txt debian@<PRIVATE_IPV4>:/tmp/"},{"line":24,"target_count":1,"text":"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \"$f\" && sudo install -m 0644 \"$f\" /var/lib/homelab-health/pve01-pushed/; done'"}],"target_cardinality":1,"target_host_is_private_ipv4":true,"target_host_sha256":"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69","target_user":"debian"},"rollback_restored":false,"rollback_started":false,"schema":15,"status":"OK","transaction_gate":{"application_architecture_found":true,"next_required_action":"build one corrected local+remote atomic transaction","pushed_health_reference_found":true,"ready_for_corrected_atomic_transaction":true,"remote_probe_root":true},"transport":{"attempts":[{"privilege":"sudo-root","rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":82754,"stdout_sha256":"603f5eb85c971d0bc029287cf91d9d0239b132dd409df5bbf8c5f47a7bb32738"}],"privilege":"sudo-root","success":true},"v14_exact_result":{"changes_made":false,"control_plane_error":null,"identity_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":10768,"output_sha256":"cc30d7f4e6be0ab5e78545d4537a39178b59e9f1a79227efaec134b78f6fc233","rollback_restored":null,"rollback_started":false,"runner_document_rc":0,"runner_json_sha256":"c932bbad4dd7f176e29621f3c549e2571baf7f84500ef2c5408a04c6a84083ff","runner_rc":0,"runner_status":"OK","runner_text_sha256":"70796c2456365c9224dd6ff02897d29412349eb1a5fea03c89ae9212217a03f8","runner_wrapper_matches_document":true,"state_document_sha256":"f3db8a869123d1266f906fdd02ca18a890141c2bd63e2bc57a3a0e7c5962f319","state_path_sha256":"2c050c8423a48aab8f448c94123e67f132381be3232e1d10274f4e87e1013b16","state_status":"OK"}}
+
+OUTPUT_END
+CHAT_OUTPUT_END
diff --git a/runtime/latest.json b/runtime/latest.json
index c2007a7..119b1d1 100644
--- a/runtime/latest.json
+++ b/runtime/latest.json
@@ -1,19 +1,19 @@
 {
   "schema_version": 1,
   "channel": "homelab-runtime",
-  "command_id": "HOMELAB-SKLADCHIK-HANDOFF-SOURCE-BIND-READ-ONLY-20260805T124700Z",
+  "command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z",
   "status": "OK",
   "rc": 0,
   "host": "pve01",
   "mode": "verify",
-  "component": "skladchik-handoff-source-bind-read-only",
-  "started_at_utc": "2026-08-05T12:50:52Z",
-  "finished_at_utc": "2026-08-05T12:50:52Z",
+  "component": "homelab-cluster-admin-update-visibility",
+  "started_at_utc": "2026-08-05T12:55:27Z",
+  "finished_at_utc": "2026-08-05T12:55:29Z",
   "reference_register_checked": true,
   "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
   "error_register_checked": true,
   "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
-  "command_sha256": "a8213c80cd21274548acfd1063b3b782d58a80e98599b151eeab8b5d53e4deb3",
+  "command_sha256": "4845b4e99e60efaa533b2a551d4bf1773085ce96ef0e4ce9737b920907b0553f",
   "duplicate_failed_command_blocked": false,
   "block_reason": null,
   "execution_started": true,
@@ -30,9 +30,9 @@
   "secrets_included": false,
   "private_addresses_included": false,
   "raw_evidence_retained_locally": true,
-  "raw_evidence_sha256": "eaa72e6c2d598300db8afc0fbb3d209fed2d940f9382077bbe5951f3f12432af",
-  "sanitized_output_sha256": "eaa72e6c2d598300db8afc0fbb3d209fed2d940f9382077bbe5951f3f12432af",
+  "raw_evidence_sha256": "b672b2fde18de662e73d8a98d4cc036895f9de2007679edc3c085392050e66e6",
+  "sanitized_output_sha256": "c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8",
   "output_truncated_in_json": false,
   "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
-  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-SKLADCHIK-HANDOFF-SOURCE-BIND-READ-ONLY-20260805T124700Z\",\"command_rc\":0,\"mutation_outcome\":\"NO_MUTATION\",\"output_truncated_in_json\":false,\"rc\":0,\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\n{\"active_files\":[{\"bytes\":665,\"mode\":\"0o644\",\"requested\":{\"basename\":\"homelab-command-adapter.service\",\"class\":\"etc\",\"path_sha256\":\"afe4d91c280ac4e6e9999498b5f5d8b8842a1f33e30fd45d984879a6392bc045\"},\"resolved\":{\"basename\":\"homelab-command-adapter.service\",\"class\":\"etc\",\"path_sha256\":\"afe4d91c280ac4e6e9999498b5f5d8b8842a1f33e30fd45d984879a6392bc045\"},\"sha256\":\"74d0f37878652ed9d46b53ae8556bbb30ebd683741f293c3f59b3527876ed2d7\",\"symlink_chain\":[{\"basename\":\"homelab-command-adapter.service\",\"class\":\"etc\",\"mode\":\"0o644\",\"path_sha256\":\"afe4d91c280ac4e6e9999498b5f5d8b8842a1f33e30fd45d984879a6392bc045\",\"symlink\":false}],\"unit\":\"homelab-command-adapter.service\"},{\"bytes\":86019,\"mode\":\"0o755\",\"python\":{\"error\":null,\"functions\":[{\"args\":[\"args\"],\"line\":658,\"name\":\"run_command\",\"segment_sha256\":\"9cc54e923af387a0d707394fca81eb7f7c23a9f0933ee69281ef97a2a9073146\"},{\"args\":[\"state\"],\"line\":1547,\"name\":\"sanitize_state\",\"segment_sha256\":\"f65062417f5cc3835ae7f91984be733e10348eb56b52d66e258abc013533e0b9\"},{\"args\":[\"argv\",\"payload_root\"],\"line\":1670,\"name\":\"substitute_argv\",\"segment_sha256\":\"0adfb439cbbe53225dac3941b6e79a6a669ea02e413e2f4134a3fd61eb82f3f0\"},{\"args\":[\"args\"],\"line\":2049,\"name\":\"run_existing_runner\",\"segment_sha256\":\"ca5964725e800420a886192740771da2099500c76ed10a7f3fd7a2e9ff4134ae\"},{\"args\":[\"request\"],\"line\":2639,\"name\":\"dispatch\",\"segment_sha256\":\"7b4d4e9ff23b1177453e51bbe7b2ce3dd10fd48f6e56e000d7ae69d6917834f4\"},{\"args\":[\"self\"],\"line\":2749,\"name\":\"handle\",\"segment_sha256\":\"ce948fe6248f462115796fcee91ec72f891339d9b603b98ce84a8842eaeb3926\"}],\"imports\":[\"__future__\",\"argparse\",\"base64\",\"datetime\",\"fcntl\",\"gzip\",\"hashlib\",\"hmac\",\"html\",\"io\",\"json\",\"os\",\"pathlib\",\"pwd\",\"re\",\"secrets\",\"shutil\",\"signal\",\"socket\",\"socketserver\",\"stat\",\"struct\",\"subprocess\",\"sys\",\"tarfile\",\"tempfile\",\"threading\",\"time\",\"typing\",\"urllib.parse\"],\"parse_ok\":true,\"process_calls\":[{\"call\":\"subprocess.run\",\"has_env\":true,\"has_stdin\":true,\"keywords\":[\"check\",\"env\",\"stderr\",\"stdin\",\"stdout\",\"text\",\"timeout\"],\"line\":664,\"segment_sha256\":\"fbe36f5d1ed7b23d545f105135de2993138aa4a8dafc9f5419d7c37b36deebae\"},{\"call\":\"subprocess.Popen\",\"has_env\":true,\"has_stdin\":true,\"keywords\":[\"close_fds\",\"env\",\"start_new_session\",\"stderr\",\"stdin\",\"stdout\"],\"line\":1626,\"segment_sha256\":\"90bb736720d2877f6704459828dc5551619ba00834a4660e9ed9c7f86e05b8c6\"},{\"call\":\"subprocess.Popen\",\"has_env\":true,\"has_stdin\":true,\"keywords\":[\"close_fds\",\"env\",\"start_new_session\",\"stderr\",\"stdin\",\"stdout\",\"text\"],\"line\":2056,\"segment_sha256\":\"4582da2d6b8e31108deedee5a5d6b877788db5da0f6d52120a605d76d0037d97\"}],\"redaction_symbols\":[\"create_evidence_bundle\",\"evidence_url\",\"remote_evidence_presign\",\"sanitize_state\",\"sign_evidence\",\"upload_evidence\"],\"request_sites\":[{\"call\":\"json.loads\",\"line\":245,\"segment_sha256\":\"6a36fbe7611c83cf1f9a144d11feb88410bbc002bb2aeeffc19cd9b97fdab2b4\"},{\"call\":\"json.loads\",\"line\":288,\"segment_sha256\":\"bf67e3bbc5d4ac5ab0d15340108bb5410b74494961d88fa35c556acbf64afbd4\"},{\"call\":\"json.loads\",\"line\":706,\"segment_sha256\":\"49c73d21d24e89e31f7701c67025900b8b000fbdde1520e1ce4bbc04d0a543c4\"},{\"call\":\"json.loads\",\"line\":2009,\"segment_sha256\":\"49c73d21d24e89e31f7701c67025900b8b000fbdde1520e1ce4bbc04d0a543c4\"},{\"call\":\"self.rfile.readline\",\"line\":2765,\"segment_sha256\":\"e42346ea97b7870f278aaf7cb87b9e20c01dd1a2e3f34f561e1fae6d4febb592\"},{\"call\":\"json.loads\",\"line\":2772,\"segment_sha256\":\"6a36fbe7611c83cf1f9a144d11feb88410bbc002bb2aeeffc19cd9b97fdab2b4\"}],\"routes\":[{\"kind\":\"startswith\",\"line\":529,\"owner\":\"\",\"path\":\"/\",\"segment_sha256\":\"b1af8a6dd713e6a332c44625432a263eab2c49721398367ee1277e38f7e8623b\"}],\"schema_fields\":[\"argv\",\"content_b64\",\"mode\",\"payload\",\"risk\",\"secret\"]},\"requested\":{\"basename\":\"homelab-command-adapter\",\"class\":\"usr_local\",\"path_sha256\":\"c2e3b43f1f858c6fa265dc276d6a6a29e0cd25e4a127fbc093b8c8d9512bb739\"},\"resolved\":{\"basename\":\"homelab-command-adapter\",\"class\":\"usr_local\",\"path_sha256\":\"c2e3b43f1f858c6fa265dc276d6a6a29e0cd25e4a127fbc093b8c8d9512bb739\"},\"sha256\":\"a1a7a615c99568bf915d084cc476f0ebf54ab0eedf3ecd578c4b0e072f58c43d\",\"symlink_chain\":[{\"basename\":\"homelab-command-adapter\",\"class\":\"usr_local\",\"mode\":\"0o755\",\"path_sha256\":\"c2e3b43f1f858c6fa265dc276d6a6a29e0cd25e4a127fbc093b8c8d9512bb739\",\"symlink\":false}],\"unit\":\"homelab-command-adapter.service\"},{\"bytes\":981,\"mode\":\"0o644\",\"requested\":{\"basename\":\"homelab-command-portal.service\",\"class\":\"etc\",\"path_sha256\":\"adf87b0f1c821b17e3e2ff19cc024d783b55caf69460d01fa66f7d2579a71f5a\"},\"resolved\":{\"basename\":\"homelab-command-portal.service\",\"class\":\"etc\",\"path_sha256\":\"adf87b0f1c821b17e3e2ff19cc024d783b55caf69460d01fa66f7d2579a71f5a\"},\"sha256\":\"8453eab30f5c3ccd285fc9f70fff18f564809bb27f7bf84f51f2ba593aa85263\",\"symlink_chain\":[{\"basename\":\"homelab-command-portal.service\",\"class\":\"etc\",\"mode\":\"0o644\",\"path_sha256\":\"adf87b0f1c821b17e3e2ff19cc024d783b55caf69460d01fa66f7d2579a71f5a\",\"symlink\":false}],\"unit\":\"homelab-command-portal.service\"},{\"bytes\":39936,\"mode\":\"0o755\",\"python\":{\"error\":null,\"functions\":[{\"args\":[\"request\"],\"line\":44,\"name\":\"adapter_call\",\"segment_sha256\":\"facf24c434f624f6a97482acbc7d053e595dc3cb1389f4b7d73ecce0f7934ac4\"},{\"args\":[\"self\",\"status\",\"content_type\",\"body\"],\"line\":471,\"name\":\"reply\",\"segment_sha256\":\"1e3a858264d98399a6e3fa9d3be77250b9afd667890f67f7172131cfc3121e8b\"},{\"args\":[\"self\"],\"line\":553,\"name\":\"do_GET\",\"segment_sha256\":\"b7332de76c63c8b6609c9e303c4b9b05943b0c0a1d544bd48a78e54a69e0f2ac\"},{\"args\":[\"self\"],\"line\":766,\"name\":\"do_POST\",\"segment_sha256\":\"11c1ce3f3c6425cbbfaf9a04211ff4025e40a8eed7628cd57bdfbee32544b035\"},{\"args\":[\"request\"],\"line\":1041,\"name\":\"fake_adapter_call\",\"segment_sha256\":\"26cdff236d21fb0a7d4799ab6de0e80dbe9104ad11836ca7ade55ecf9f66c42d\"}],\"imports\":[\"__future__\",\"argparse\",\"hashlib\",\"hmac\",\"html\",\"http.cookies\",\"http.server\",\"ipaddress\",\"json\",\"os\",\"pathlib\",\"re\",\"secrets\",\"socket\",\"sys\",\"time\",\"typing\",\"urllib.parse\"],\"parse_ok\":true,\"process_calls\":[],\"redaction_symbols\":[],\"request_sites\":[{\"call\":\"json.loads\",\"line\":81,\"segment_sha256\":\"6a36fbe7611c83cf1f9a144d11feb88410bbc002bb2aeeffc19cd9b97fdab2b4\"},{\"call\":\"self.rfile.read\",\"line\":541,\"segment_sha256\":\"e88ccc84bddb9730bf85279cc75080f708954fea1522420d79deeef7f4d4022e\"},{\"call\":\"urllib.parse.parse_qs\",\"line\":548,\"segment_sha256\":\"db330b7137c377ba69515c420495114e333999d1d4f22ed4a476b59e8130c937\"},{\"call\":\"urllib.parse.parse_qs\",\"line\":852,\"segment_sha256\":\"24f22df8901b81aff82756ae2c8010bb392d60f2d336a6e75562a87c7bfd5adb\"}],\"routes\":[{\"kind\":\"compare\",\"line\":596,\"owner\":\"\",\"path\":\"/bridge/bridge.cmd\",\"segment_sha256\":\"bea803e2bf945e9c5008a239e11936ecdbd8475018029010553aae5d638af370\"},{\"kind\":\"startswith\",\"line\":609,\"owner\":\"\",\"path\":\"/enroll/\",\"segment_sha256\":\"902d0c2af350b77c90fcd32dbf60dd2362c9e0d7ec504ad185488cc97161ef56\"},{\"kind\":\"compare\",\"line\":771,\"owner\":\"\",\"path\":\"/api/enroll\",\"segment_sha256\":\"d1e94b164fcc17d1db52d30967b862e840416d96902d98986ba7d831796985dd\"},{\"kind\":\"compare\",\"line\":791,\"owner\":\"\",\"path\":\"/api/import\",\"segment_sha256\":\"528504d107f4e3a9c88fb94943e594ed199c19d14b7398d0499ed62a41c176f9\"}],\"schema_fields\":[\"Authorization\",\"Cookie\",\"argv\",\"mode\",\"payload\",\"risk\"]},\"requested\":{\"basename\":\"homelab-command-portal\",\"class\":\"usr_local\",\"path_sha256\":\"1132c3c355ed18cdc6406935cd26a128df298f71df7f48df690c72b202ddd7be\"},\"resolved\":{\"basename\":\"homelab-command-portal\",\"class\":\"usr_local\",\"path_sha256\":\"1132c3c355ed18cdc6406935cd26a128df298f71df7f48df690c72b202ddd7be\"},\"sha256\":\"97c69bd4fccf7272dadc911281c1503f411bae2dc11dac4b22b87f33d678ae52\",\"symlink_chain\":[{\"basename\":\"homelab-command-portal\",\"class\":\"usr_local\",\"mode\":\"0o755\",\"path_sha256\":\"1132c3c355ed18cdc6406935cd26a128df298f71df7f48df690c72b202ddd7be\",\"symlink\":false}],\"unit\":\"homelab-command-portal.service\"}],\"analysis_ready\":true,\"changes_made\":false,\"command_id\":\"HOMELAB-SKLADCHIK-HANDOFF-SOURCE-BIND-READ-ONLY-20260805T124700Z\",\"command_rc\":0,\"git\":{\"anchor\":{\"basename\":\"cr-2026-0015\",\"class\":\"srv\",\"path_sha256\":\"1b3beda8fc19beb489d31ac23276ec633dbe86b73185a75cab96cb5fe244fc9f\"},\"anchor_present\":true,\"provenance\":{\"homelab-command-adapter\":{\"active_sha256\":\"a1a7a615c99568bf915d084cc476f0ebf54ab0eedf3ecd578c4b0e072f58c43d\",\"current_worktree_matches\":[],\"exact_source_bound\":false,\"history_matches\":[]},\"homelab-command-portal\":{\"active_sha256\":\"97c69bd4fccf7272dadc911281c1503f411bae2dc11dac4b22b87f33d678ae52\",\"current_worktree_matches\":[],\"exact_source_bound\":false,\"history_matches\":[]}},\"worktree_count\":3},\"mutation_outcome\":\"NO_MUTATION\",\"next_action\":\"BUILD_SHA_GUARDED_EPHEMERAL_SECRET_HANDOFF_TRANSACTION\",\"output_truncated_in_json\":false,\"rc\":0,\"read_only\":true,\"readiness\":{\"adapter_ready\":true,\"exact_files_ready\":true,\"portal_ready\":true,\"source_bound\":false},\"rollback_restored\":null,\"rollback_started\":false,\"root_cause_classification\":\"ACTIVE_EXECUTION_VERIFIED_GIT_SOURCE_UNBOUND\",\"schema\":1,\"source_bound\":false,\"status\":\"OK\",\"units\":[{\"active_state\":\"active\",\"environment_files_declared\":false,\"show_rc\":0,\"sources\":[{\"basename\":\"homelab-command-adapter.service\",\"class\":\"etc\",\"path_sha256\":\"afe4d91c280ac4e6e9999498b5f5d8b8842a1f33e30fd45d984879a6392bc045\"},{\"basename\":\"homelab-command-adapter\",\"class\":\"usr_local\",\"path_sha256\":\"c2e3b43f1f858c6fa265dc276d6a6a29e0cd25e4a127fbc093b8c8d9512bb739\"}],\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"sub_state\":\"running\",\"unit\":\"homelab-command-adapter.service\",\"user_class\":\"root\",\"working_directory_declared\":false},{\"active_state\":\"active\",\"environment_files_declared\":false,\"show_rc\":0,\"sources\":[{\"basename\":\"homelab-command-portal.service\",\"class\":\"etc\",\"path_sha256\":\"adf87b0f1c821b17e3e2ff19cc024d783b55caf69460d01fa66f7d2579a71f5a\"},{\"basename\":\"homelab-command-portal\",\"class\":\"usr_local\",\"path_sha256\":\"1132c3c355ed18cdc6406935cd26a128df298f71df7f48df690c72b202ddd7be\"}],\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"sub_state\":\"running\",\"unit\":\"homelab-command-portal.service\",\"user_class\":\"non_root\",\"working_directory_declared\":false}]}\n"
+  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z\",\"command_rc\":0,\"control_plane_error\":null,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_corrected_atomic_transaction\":true,\"remote_application_candidate_count\":40,\"remote_hostname\":\"cluster-admin\",\"remote_index_count\":1,\"remote_probe_privilege\":\"sudo-root\",\"remote_reference_file_count\":1,\"remote_target_cardinality\":1,\"remote_target_host_sha256\":\"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69\",\"remote_unit_count\":16,\"remote_webroot_config_count\":0,\"report_bytes\":85056,\"report_sha256\":\"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"v14_exact_result_identity_confirmed\":true,\"version\":1,\"webpanel_script_sha256\":\"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276\"}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z\",\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"remote_architecture\":{\"application_candidates\":[{\"bytes\":133,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},{\"bytes\":149,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},{\"bytes\":151,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},{\"bytes\":231,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},{\"bytes\":138,\"matches\":[{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},{\"bytes\":127,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},{\"bytes\":616,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},{\"bytes\":1190,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},{\"bytes\":223,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},{\"bytes\":803,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},{\"bytes\":131,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},{\"bytes\":301,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},{\"bytes\":89,\"matches\":[{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\"},{\"bytes\":8444,\"matches\":[{\"line\":4,\"text\":\"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')\"},{\"line\":64,\"text\":\" return f\\\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\\\n\\\"\"},{\"line\":82,\"text\":\" old=<REDACTED>\"},{\"line\":97,\"text\":\"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)\"},{\"line\":99,\"text\":\"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\"},{\"bytes\":28599,\"matches\":[{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =<REDACTED>\"},{\"line\":14,\"text\":\"COOKIE =<REDACTED>\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":20,\"text\":\"    return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":73,\"text\":\"    return f\\\"\\\"\\\"<!doctype html><html lang=<REDACTED>\"},{\"line\":152,\"text\":\"    return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":220,\"text\":\"    page = f\\\"\\\"\\\"<!doctype html><html lang=\\\"ru\\\"><head><meta charset=\\\"utf-8\\\"><meta name=\\\"viewport\\\" content=\\\"width=device-width,initial-scale=1\\\"><meta http-equiv=\\\"refresh\\\" content=\\\"30\\\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\\\"post\\\" action=\\\"/logout\\\"><input type=\\\"hidden\\\" name=\\\"csrf\\\" value=\\\"{csrf}\\\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\\\"\\\"\\\"\"},{\"line\":243,\"text\":\"        \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\"        \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\"        \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\"        \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\"        \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\"        \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\"        \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\"        \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\"        \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\"        \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\"        \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\"        \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\"        \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\"        \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\"        \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\"        \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\"        \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\"        \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\"        \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\"        \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\"        \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":267,\"text\":\"        lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\"},{\"bytes\":922,\"matches\":[{\"line\":2,\"text\":\"first = getpass.getpass(\\\"Новый пароль Cluster Admin: \\\")\"},{\"line\":12,\"text\":\"directory = \\\"/etc/cluster-admin-incident-engine\\\"\"},{\"line\":16,\"text\":\"account = pwd.getpwnam(\\\"clusteradmin\\\")\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\"},{\"bytes\":10640,\"matches\":[{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":36,\"text\":\"    data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\"    connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":41,\"text\":\"        cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":50,\"text\":\"            if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":57,\"text\":\"        cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":108,\"text\":\"    line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\"},{\"bytes\":1857,\"matches\":[{\"line\":3,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage1-foundation-migration-plan\\\",\"},{\"line\":43,\"text\":\"    \\\"function\\\": \\\"cluster_admin_enrich_collector_run\\\",\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\"},{\"bytes\":1342,\"matches\":[{\"line\":3,\"text\":\"  \\\"collector_call\\\": \\\"SELECT cluster_admin_sync_foundation(%s)\\\",\"},{\"line\":15,\"text\":\"  \\\"function\\\": \\\"cluster_admin_sync_foundation\\\",\"},{\"line\":16,\"text\":\"  \\\"function_signature\\\": \\\"cluster_admin_sync_foundation(bigint)\\\",\"},{\"line\":26,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-migration-plan\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\"},{\"bytes\":2355,\"matches\":[{\"line\":51,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-dynamic-acceptance-contract\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\"},{\"bytes\":2832,\"matches\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\"  line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\"  echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},{\"bytes\":3471,\"matches\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\"  echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\"  echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},{\"bytes\":1433,\"matches\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},{\"bytes\":848,\"matches\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":10353,\"matches\":[{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\"},{\"bytes\":168,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729\"},{\"bytes\":259,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741\"},{\"bytes\":134,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517\"},{\"bytes\":141,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\"},{\"bytes\":189,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\"},{\"bytes\":131,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":199,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\"},{\"bytes\":272,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec\"},{\"bytes\":380,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"}],\"euid\":0,\"gate\":{\"application_candidate_count\":40,\"ready_for_remote_transaction_design\":true,\"reference_file_count\":1,\"unit_count\":16},\"health_input_files\":[{\"bytes\":380,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"},{\"bytes\":128,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\"},{\"bytes\":1535,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\"}],\"hostname\":\"cluster-admin\",\"indexes\":[{\"bytes\":10353,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"root\":\"/var/www/homelab-cluster-admin\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\",\"title_lines\":[\"<title>Homelab Cluster Admin</title>\",\"<h1>Homelab Cluster Admin</h1>\",\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\",\"<div class=<REDACTED>\",\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\",\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"]}],\"listeners\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"State  Recv-Q Send-Q Local Address:Port Peer Address:PortProcess                                   \\nLISTEN 0      5            0.0.0.0:8080      0.0.0.0:*    users:((\\\"python3\\\",pid=439,fd=3))         \\nLISTEN 0      2048         0.0.0.0:8081      0.0.0.0:*    users:((\\\"python3\\\",pid=487,fd=6))         \\nLISTEN 0      244        127.0.0.1:5432      0.0.0.0:*    users:((\\\"postgres\\\",pid=469,fd=6))        \\nLISTEN 0      4096   127.0.0.53%lo:53        0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=18))\\nLISTEN 0      4096         0.0.0.0:5355      0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=12))\\nLISTEN 0      4096      127.0.0.54:53        0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=20))\\nLISTEN 0      128          0.0.0.0:22        0.0.0.0:*    users:((\\\"sshd\\\",pid=457,fd=3))            \\nLISTEN 0      244            [::1]:5432         [::]:*    users:((\\\"postgres\\\",pid=469,fd=5))        \\nLISTEN 0      4096            [::]:5355         [::]:*    users:((\\\"systemd-resolve\\\",pid=380,fd=14))\\nLISTEN 0      128             [::]:22           [::]:*    users:((\\\"sshd\\\",pid=457,fd=4))            \"},\"matching_files\":[{\"bytes\":133,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},{\"bytes\":149,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},{\"bytes\":151,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},{\"bytes\":231,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},{\"bytes\":138,\"matches\":[{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},{\"bytes\":127,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},{\"bytes\":616,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},{\"bytes\":1190,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},{\"bytes\":223,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},{\"bytes\":803,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},{\"bytes\":131,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},{\"bytes\":301,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},{\"bytes\":89,\"matches\":[{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\"},{\"bytes\":8444,\"matches\":[{\"line\":4,\"text\":\"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')\"},{\"line\":64,\"text\":\" return f\\\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\\\n\\\"\"},{\"line\":82,\"text\":\" old=<REDACTED>\"},{\"line\":97,\"text\":\"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)\"},{\"line\":99,\"text\":\"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\"},{\"bytes\":28599,\"matches\":[{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =<REDACTED>\"},{\"line\":14,\"text\":\"COOKIE =<REDACTED>\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":20,\"text\":\"    return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":73,\"text\":\"    return f\\\"\\\"\\\"<!doctype html><html lang=<REDACTED>\"},{\"line\":152,\"text\":\"    return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":220,\"text\":\"    page = f\\\"\\\"\\\"<!doctype html><html lang=\\\"ru\\\"><head><meta charset=\\\"utf-8\\\"><meta name=\\\"viewport\\\" content=\\\"width=device-width,initial-scale=1\\\"><meta http-equiv=\\\"refresh\\\" content=\\\"30\\\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\\\"post\\\" action=\\\"/logout\\\"><input type=\\\"hidden\\\" name=\\\"csrf\\\" value=\\\"{csrf}\\\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\\\"\\\"\\\"\"},{\"line\":243,\"text\":\"        \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\"        \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\"        \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\"        \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\"        \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\"        \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\"        \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\"        \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\"        \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\"        \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\"        \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\"        \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\"        \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\"        \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\"        \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\"        \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\"        \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\"        \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\"        \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\"        \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\"        \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":267,\"text\":\"        lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\"},{\"bytes\":922,\"matches\":[{\"line\":2,\"text\":\"first = getpass.getpass(\\\"Новый пароль Cluster Admin: \\\")\"},{\"line\":12,\"text\":\"directory = \\\"/etc/cluster-admin-incident-engine\\\"\"},{\"line\":16,\"text\":\"account = pwd.getpwnam(\\\"clusteradmin\\\")\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\"},{\"bytes\":10640,\"matches\":[{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":36,\"text\":\"    data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\"    connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":41,\"text\":\"        cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":50,\"text\":\"            if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":57,\"text\":\"        cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":108,\"text\":\"    line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\"},{\"bytes\":1857,\"matches\":[{\"line\":3,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage1-foundation-migration-plan\\\",\"},{\"line\":43,\"text\":\"    \\\"function\\\": \\\"cluster_admin_enrich_collector_run\\\",\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\"},{\"bytes\":1342,\"matches\":[{\"line\":3,\"text\":\"  \\\"collector_call\\\": \\\"SELECT cluster_admin_sync_foundation(%s)\\\",\"},{\"line\":15,\"text\":\"  \\\"function\\\": \\\"cluster_admin_sync_foundation\\\",\"},{\"line\":16,\"text\":\"  \\\"function_signature\\\": \\\"cluster_admin_sync_foundation(bigint)\\\",\"},{\"line\":26,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-migration-plan\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\"},{\"bytes\":2355,\"matches\":[{\"line\":51,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-dynamic-acceptance-contract\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\"},{\"bytes\":2832,\"matches\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\"  line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\"  echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},{\"bytes\":3471,\"matches\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\"  echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\"  echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},{\"bytes\":1433,\"matches\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},{\"bytes\":848,\"matches\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":10353,\"matches\":[{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\"},{\"bytes\":168,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729\"},{\"bytes\":259,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741\"},{\"bytes\":134,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517\"},{\"bytes\":141,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\"},{\"bytes\":189,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\"},{\"bytes\":131,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":199,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\"},{\"bytes\":272,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec\"},{\"bytes\":380,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":1535,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\"},{\"bytes\":128,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\"},{\"bytes\":217,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"}],\"process_lines\":[\"    439       1 root     root     python3         /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\",\"    487       1 cluster+ cluster+ python3         /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\"],\"proxies\":[],\"pve01_pushed_reference_files\":[{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"}],\"scan\":{\"directories_scanned\":68,\"index_count\":1,\"match_count\":42},\"units\":[{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"Group\":\"root\",\"LoadState\":\"loaded\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"root\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}],\"webroots\":[]},\"remote_target\":{\"evidence\":[{\"line\":21,\"target_count\":1,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$TMP\\\"/*.txt debian@<PRIVATE_IPV4>:/tmp/\"},{\"line\":24,\"target_count\":1,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \\\"$f\\\" && sudo install -m 0644 \\\"$f\\\" /var/lib/homelab-health/pve01-pushed/; done'\"}],\"target_cardinality\":1,\"target_host_is_private_ipv4\":true,\"target_host_sha256\":\"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69\",\"target_user\":\"debian\"},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":15,\"status\":\"OK\",\"transaction_gate\":{\"application_architecture_found\":true,\"next_required_action\":\"build one corrected local+remote atomic transaction\",\"pushed_health_reference_found\":true,\"ready_for_corrected_atomic_transaction\":true,\"remote_probe_root\":true},\"transport\":{\"attempts\":[{\"privilege\":\"sudo-root\",\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":82754,\"stdout_sha256\":\"603f5eb85c971d0bc029287cf91d9d0239b132dd409df5bbf8c5f47a7bb32738\"}],\"privilege\":\"sudo-root\",\"success\":true},\"v14_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":10768,\"output_sha256\":\"cc30d7f4e6be0ab5e78545d4537a39178b59e9f1a79227efaec134b78f6fc233\",\"rollback_restored\":null,\"rollback_started\":false,\"runner_document_rc\":0,\"runner_json_sha256\":\"c932bbad4dd7f176e29621f3c549e2571baf7f84500ef2c5408a04c6a84083ff\",\"runner_rc\":0,\"runner_status\":\"OK\",\"runner_text_sha256\":\"70796c2456365c9224dd6ff02897d29412349eb1a5fea03c89ae9212217a03f8\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"f3db8a869123d1266f906fdd02ca18a890141c2bd63e2bc57a3a0e7c5962f319\",\"state_path_sha256\":\"2c050c8423a48aab8f448c94123e67f132381be3232e1d10274f4e87e1013b16\",\"state_status\":\"OK\"}}\n"
 }
diff --git a/runtime/latest.txt b/runtime/latest.txt
index 52c7946..04c4bc8 100644
--- a/runtime/latest.txt
+++ b/runtime/latest.txt
@@ -1,15 +1,15 @@
 CHAT_OUTPUT_BEGIN
-COMMAND_ID=HOMELAB-SKLADCHIK-HANDOFF-SOURCE-BIND-READ-ONLY-20260805T124700Z
+COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z
 STATUS=OK
 RC=0
 HOST=pve01
 MODE=verify
-COMPONENT=skladchik-handoff-source-bind-read-only
+COMPONENT=homelab-cluster-admin-update-visibility
 REFERENCE_REGISTER_CHECK=OK
 REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
 ERROR_REGISTER_CHECK=OK
 ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
-COMMAND_SHA256=a8213c80cd21274548acfd1063b3b782d58a80e98599b151eeab8b5d53e4deb3
+COMMAND_SHA256=4845b4e99e60efaa533b2a551d4bf1773085ce96ef0e4ce9737b920907b0553f
 DUPLICATE_FAILED_COMMAND_BLOCKED=false
 EXECUTION_STARTED=true
 CHANGE_DECLARED=false
@@ -24,11 +24,11 @@ MUTATION_OUTCOME=NO_MUTATION
 SANITIZED=yes
 SECRETS_INCLUDED=no
 PRIVATE_ADDRESSES_INCLUDED=no
-RAW_EVIDENCE_SHA256=eaa72e6c2d598300db8afc0fbb3d209fed2d940f9382077bbe5951f3f12432af
-SANITIZED_OUTPUT_SHA256=eaa72e6c2d598300db8afc0fbb3d209fed2d940f9382077bbe5951f3f12432af
+RAW_EVIDENCE_SHA256=b672b2fde18de662e73d8a98d4cc036895f9de2007679edc3c085392050e66e6
+SANITIZED_OUTPUT_SHA256=c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8
 OUTPUT_BEGIN
-HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-SKLADCHIK-HANDOFF-SOURCE-BIND-READ-ONLY-20260805T124700Z","command_rc":0,"mutation_outcome":"NO_MUTATION","output_truncated_in_json":false,"rc":0,"rollback_restored":null,"rollback_started":false,"status":"OK","version":1}
-{"active_files":[{"bytes":665,"mode":"0o644","requested":{"basename":"homelab-command-adapter.service","class":"etc","path_sha256":"afe4d91c280ac4e6e9999498b5f5d8b8842a1f33e30fd45d984879a6392bc045"},"resolved":{"basename":"homelab-command-adapter.service","class":"etc","path_sha256":"afe4d91c280ac4e6e9999498b5f5d8b8842a1f33e30fd45d984879a6392bc045"},"sha256":"74d0f37878652ed9d46b53ae8556bbb30ebd683741f293c3f59b3527876ed2d7","symlink_chain":[{"basename":"homelab-command-adapter.service","class":"etc","mode":"0o644","path_sha256":"afe4d91c280ac4e6e9999498b5f5d8b8842a1f33e30fd45d984879a6392bc045","symlink":false}],"unit":"homelab-command-adapter.service"},{"bytes":86019,"mode":"0o755","python":{"error":null,"functions":[{"args":["args"],"line":658,"name":"run_command","segment_sha256":"9cc54e923af387a0d707394fca81eb7f7c23a9f0933ee69281ef97a2a9073146"},{"args":["state"],"line":1547,"name":"sanitize_state","segment_sha256":"f65062417f5cc3835ae7f91984be733e10348eb56b52d66e258abc013533e0b9"},{"args":["argv","payload_root"],"line":1670,"name":"substitute_argv","segment_sha256":"0adfb439cbbe53225dac3941b6e79a6a669ea02e413e2f4134a3fd61eb82f3f0"},{"args":["args"],"line":2049,"name":"run_existing_runner","segment_sha256":"ca5964725e800420a886192740771da2099500c76ed10a7f3fd7a2e9ff4134ae"},{"args":["request"],"line":2639,"name":"dispatch","segment_sha256":"7b4d4e9ff23b1177453e51bbe7b2ce3dd10fd48f6e56e000d7ae69d6917834f4"},{"args":["self"],"line":2749,"name":"handle","segment_sha256":"ce948fe6248f462115796fcee91ec72f891339d9b603b98ce84a8842eaeb3926"}],"imports":["__future__","argparse","base64","datetime","fcntl","gzip","hashlib","hmac","html","io","json","os","pathlib","pwd","re","secrets","shutil","signal","socket","socketserver","stat","struct","subprocess","sys","tarfile","tempfile","threading","time","typing","urllib.parse"],"parse_ok":true,"process_calls":[{"call":"subprocess.run","has_env":true,"has_stdin":true,"keywords":["check","env","stderr","stdin","stdout","text","timeout"],"line":664,"segment_sha256":"fbe36f5d1ed7b23d545f105135de2993138aa4a8dafc9f5419d7c37b36deebae"},{"call":"subprocess.Popen","has_env":true,"has_stdin":true,"keywords":["close_fds","env","start_new_session","stderr","stdin","stdout"],"line":1626,"segment_sha256":"90bb736720d2877f6704459828dc5551619ba00834a4660e9ed9c7f86e05b8c6"},{"call":"subprocess.Popen","has_env":true,"has_stdin":true,"keywords":["close_fds","env","start_new_session","stderr","stdin","stdout","text"],"line":2056,"segment_sha256":"4582da2d6b8e31108deedee5a5d6b877788db5da0f6d52120a605d76d0037d97"}],"redaction_symbols":["create_evidence_bundle","evidence_url","remote_evidence_presign","sanitize_state","sign_evidence","upload_evidence"],"request_sites":[{"call":"json.loads","line":245,"segment_sha256":"6a36fbe7611c83cf1f9a144d11feb88410bbc002bb2aeeffc19cd9b97fdab2b4"},{"call":"json.loads","line":288,"segment_sha256":"bf67e3bbc5d4ac5ab0d15340108bb5410b74494961d88fa35c556acbf64afbd4"},{"call":"json.loads","line":706,"segment_sha256":"49c73d21d24e89e31f7701c67025900b8b000fbdde1520e1ce4bbc04d0a543c4"},{"call":"json.loads","line":2009,"segment_sha256":"49c73d21d24e89e31f7701c67025900b8b000fbdde1520e1ce4bbc04d0a543c4"},{"call":"self.rfile.readline","line":2765,"segment_sha256":"e42346ea97b7870f278aaf7cb87b9e20c01dd1a2e3f34f561e1fae6d4febb592"},{"call":"json.loads","line":2772,"segment_sha256":"6a36fbe7611c83cf1f9a144d11feb88410bbc002bb2aeeffc19cd9b97fdab2b4"}],"routes":[{"kind":"startswith","line":529,"owner":"","path":"/","segment_sha256":"b1af8a6dd713e6a332c44625432a263eab2c49721398367ee1277e38f7e8623b"}],"schema_fields":["argv","content_b64","mode","payload","risk","secret"]},"requested":{"basename":"homelab-command-adapter","class":"usr_local","path_sha256":"c2e3b43f1f858c6fa265dc276d6a6a29e0cd25e4a127fbc093b8c8d9512bb739"},"resolved":{"basename":"homelab-command-adapter","class":"usr_local","path_sha256":"c2e3b43f1f858c6fa265dc276d6a6a29e0cd25e4a127fbc093b8c8d9512bb739"},"sha256":"a1a7a615c99568bf915d084cc476f0ebf54ab0eedf3ecd578c4b0e072f58c43d","symlink_chain":[{"basename":"homelab-command-adapter","class":"usr_local","mode":"0o755","path_sha256":"c2e3b43f1f858c6fa265dc276d6a6a29e0cd25e4a127fbc093b8c8d9512bb739","symlink":false}],"unit":"homelab-command-adapter.service"},{"bytes":981,"mode":"0o644","requested":{"basename":"homelab-command-portal.service","class":"etc","path_sha256":"adf87b0f1c821b17e3e2ff19cc024d783b55caf69460d01fa66f7d2579a71f5a"},"resolved":{"basename":"homelab-command-portal.service","class":"etc","path_sha256":"adf87b0f1c821b17e3e2ff19cc024d783b55caf69460d01fa66f7d2579a71f5a"},"sha256":"8453eab30f5c3ccd285fc9f70fff18f564809bb27f7bf84f51f2ba593aa85263","symlink_chain":[{"basename":"homelab-command-portal.service","class":"etc","mode":"0o644","path_sha256":"adf87b0f1c821b17e3e2ff19cc024d783b55caf69460d01fa66f7d2579a71f5a","symlink":false}],"unit":"homelab-command-portal.service"},{"bytes":39936,"mode":"0o755","python":{"error":null,"functions":[{"args":["request"],"line":44,"name":"adapter_call","segment_sha256":"facf24c434f624f6a97482acbc7d053e595dc3cb1389f4b7d73ecce0f7934ac4"},{"args":["self","status","content_type","body"],"line":471,"name":"reply","segment_sha256":"1e3a858264d98399a6e3fa9d3be77250b9afd667890f67f7172131cfc3121e8b"},{"args":["self"],"line":553,"name":"do_GET","segment_sha256":"b7332de76c63c8b6609c9e303c4b9b05943b0c0a1d544bd48a78e54a69e0f2ac"},{"args":["self"],"line":766,"name":"do_POST","segment_sha256":"11c1ce3f3c6425cbbfaf9a04211ff4025e40a8eed7628cd57bdfbee32544b035"},{"args":["request"],"line":1041,"name":"fake_adapter_call","segment_sha256":"26cdff236d21fb0a7d4799ab6de0e80dbe9104ad11836ca7ade55ecf9f66c42d"}],"imports":["__future__","argparse","hashlib","hmac","html","http.cookies","http.server","ipaddress","json","os","pathlib","re","secrets","socket","sys","time","typing","urllib.parse"],"parse_ok":true,"process_calls":[],"redaction_symbols":[],"request_sites":[{"call":"json.loads","line":81,"segment_sha256":"6a36fbe7611c83cf1f9a144d11feb88410bbc002bb2aeeffc19cd9b97fdab2b4"},{"call":"self.rfile.read","line":541,"segment_sha256":"e88ccc84bddb9730bf85279cc75080f708954fea1522420d79deeef7f4d4022e"},{"call":"urllib.parse.parse_qs","line":548,"segment_sha256":"db330b7137c377ba69515c420495114e333999d1d4f22ed4a476b59e8130c937"},{"call":"urllib.parse.parse_qs","line":852,"segment_sha256":"24f22df8901b81aff82756ae2c8010bb392d60f2d336a6e75562a87c7bfd5adb"}],"routes":[{"kind":"compare","line":596,"owner":"","path":"/bridge/bridge.cmd","segment_sha256":"bea803e2bf945e9c5008a239e11936ecdbd8475018029010553aae5d638af370"},{"kind":"startswith","line":609,"owner":"","path":"/enroll/","segment_sha256":"902d0c2af350b77c90fcd32dbf60dd2362c9e0d7ec504ad185488cc97161ef56"},{"kind":"compare","line":771,"owner":"","path":"/api/enroll","segment_sha256":"d1e94b164fcc17d1db52d30967b862e840416d96902d98986ba7d831796985dd"},{"kind":"compare","line":791,"owner":"","path":"/api/import","segment_sha256":"528504d107f4e3a9c88fb94943e594ed199c19d14b7398d0499ed62a41c176f9"}],"schema_fields":["Authorization","Cookie","argv","mode","payload","risk"]},"requested":{"basename":"homelab-command-portal","class":"usr_local","path_sha256":"1132c3c355ed18cdc6406935cd26a128df298f71df7f48df690c72b202ddd7be"},"resolved":{"basename":"homelab-command-portal","class":"usr_local","path_sha256":"1132c3c355ed18cdc6406935cd26a128df298f71df7f48df690c72b202ddd7be"},"sha256":"97c69bd4fccf7272dadc911281c1503f411bae2dc11dac4b22b87f33d678ae52","symlink_chain":[{"basename":"homelab-command-portal","class":"usr_local","mode":"0o755","path_sha256":"1132c3c355ed18cdc6406935cd26a128df298f71df7f48df690c72b202ddd7be","symlink":false}],"unit":"homelab-command-portal.service"}],"analysis_ready":true,"changes_made":false,"command_id":"HOMELAB-SKLADCHIK-HANDOFF-SOURCE-BIND-READ-ONLY-20260805T124700Z","command_rc":0,"git":{"anchor":{"basename":"cr-2026-0015","class":"srv","path_sha256":"1b3beda8fc19beb489d31ac23276ec633dbe86b73185a75cab96cb5fe244fc9f"},"anchor_present":true,"provenance":{"homelab-command-adapter":{"active_sha256":"a1a7a615c99568bf915d084cc476f0ebf54ab0eedf3ecd578c4b0e072f58c43d","current_worktree_matches":[],"exact_source_bound":false,"history_matches":[]},"homelab-command-portal":{"active_sha256":"97c69bd4fccf7272dadc911281c1503f411bae2dc11dac4b22b87f33d678ae52","current_worktree_matches":[],"exact_source_bound":false,"history_matches":[]}},"worktree_count":3},"mutation_outcome":"NO_MUTATION","next_action":"BUILD_SHA_GUARDED_EPHEMERAL_SECRET_HANDOFF_TRANSACTION","output_truncated_in_json":false,"rc":0,"read_only":true,"readiness":{"adapter_ready":true,"exact_files_ready":true,"portal_ready":true,"source_bound":false},"rollback_restored":null,"rollback_started":false,"root_cause_classification":"ACTIVE_EXECUTION_VERIFIED_GIT_SOURCE_UNBOUND","schema":1,"source_bound":false,"status":"OK","units":[{"active_state":"active","environment_files_declared":false,"show_rc":0,"sources":[{"basename":"homelab-command-adapter.service","class":"etc","path_sha256":"afe4d91c280ac4e6e9999498b5f5d8b8842a1f33e30fd45d984879a6392bc045"},{"basename":"homelab-command-adapter","class":"usr_local","path_sha256":"c2e3b43f1f858c6fa265dc276d6a6a29e0cd25e4a127fbc093b8c8d9512bb739"}],"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","sub_state":"running","unit":"homelab-command-adapter.service","user_class":"root","working_directory_declared":false},{"active_state":"active","environment_files_declared":false,"show_rc":0,"sources":[{"basename":"homelab-command-portal.service","class":"etc","path_sha256":"adf87b0f1c821b17e3e2ff19cc024d783b55caf69460d01fa66f7d2579a71f5a"},{"basename":"homelab-command-portal","class":"usr_local","path_sha256":"1132c3c355ed18cdc6406935cd26a128df298f71df7f48df690c72b202ddd7be"}],"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","sub_state":"running","unit":"homelab-command-portal.service","user_class":"non_root","working_directory_declared":false}]}
+HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z","command_rc":0,"control_plane_error":null,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_corrected_atomic_transaction":true,"remote_application_candidate_count":40,"remote_hostname":"cluster-admin","remote_index_count":1,"remote_probe_privilege":"sudo-root","remote_reference_file_count":1,"remote_target_cardinality":1,"remote_target_host_sha256":"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69","remote_unit_count":16,"remote_webroot_config_count":0,"report_bytes":85056,"report_sha256":"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf","rollback_restored":false,"rollback_started":false,"status":"OK","v14_exact_result_identity_confirmed":true,"version":1,"webpanel_script_sha256":"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276"}
+{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z","mutation_outcome":"NO_MUTATION","read_only":true,"remote_architecture":{"application_candidates":[{"bytes":133,"matches":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},{"bytes":149,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},{"bytes":154,"matches":[{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},{"bytes":151,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},{"bytes":231,"matches":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},{"bytes":154,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},{"bytes":138,"matches":[{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},{"bytes":127,"matches":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},{"bytes":616,"matches":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},{"bytes":1190,"matches":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},{"bytes":223,"matches":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},{"bytes":803,"matches":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},{"bytes":131,"matches":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},{"bytes":301,"matches":[{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},{"bytes":89,"matches":[{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f"},{"bytes":8444,"matches":[{"line":4,"text":"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')"},{"line":64,"text":" return f\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\n\""},{"line":82,"text":" old=<REDACTED>"},{"line":97,"text":"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)"},{"line":99,"text":"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))"}],"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a"},{"bytes":28599,"matches":[{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE =<REDACTED>"},{"line":14,"text":"COOKIE =<REDACTED>"},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":20,"text":"    return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":73,"text":"    return f\"\"\"<!doctype html><html lang=<REDACTED>"},{"line":152,"text":"    return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":220,"text":"    page = f\"\"\"<!doctype html><html lang=\"ru\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta http-equiv=\"refresh\" content=\"30\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\"post\" action=\"/logout\"><input type=\"hidden\" name=\"csrf\" value=\"{csrf}\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\"\"\""},{"line":243,"text":"        \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":"        \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":"        \"cluster_admin_up 1\","},{"line":246,"text":"        \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":"        \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":"        \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":"        \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":"        \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":"        \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":"        \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":"        \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":"        \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":"        \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":"        \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":"        \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":"        \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":"        \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":"        \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":"        \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":"        \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":"        \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":267,"text":"        lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6"},{"bytes":922,"matches":[{"line":2,"text":"first = getpass.getpass(\"Новый пароль Cluster Admin: \")"},{"line":12,"text":"directory = \"/etc/cluster-admin-incident-engine\""},{"line":16,"text":"account = pwd.getpwnam(\"clusteradmin\")"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3"},{"bytes":10640,"matches":[{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":36,"text":"    data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":"    connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":41,"text":"        cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":50,"text":"            if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":57,"text":"        cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":108,"text":"    line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe"},{"bytes":1857,"matches":[{"line":3,"text":"  \"type\": \"cluster-admin-stage1-foundation-migration-plan\","},{"line":43,"text":"    \"function\": \"cluster_admin_enrich_collector_run\","}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670"},{"bytes":1342,"matches":[{"line":3,"text":"  \"collector_call\": \"SELECT cluster_admin_sync_foundation(%s)\","},{"line":15,"text":"  \"function\": \"cluster_admin_sync_foundation\","},{"line":16,"text":"  \"function_signature\": \"cluster_admin_sync_foundation(bigint)\","},{"line":26,"text":"  \"type\": \"cluster-admin-stage2-migration-plan\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c"},{"bytes":2355,"matches":[{"line":51,"text":"  \"type\": \"cluster-admin-stage2-dynamic-acceptance-contract\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588"},{"bytes":2832,"matches":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},{"bytes":3471,"matches":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},{"bytes":1433,"matches":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},{"bytes":848,"matches":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":10353,"matches":[{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a"},{"bytes":168,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify"}],"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729"},{"bytes":259,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741"},{"bytes":134,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517"},{"bytes":141,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6"},{"bytes":189,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941"},{"bytes":131,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0"}],"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":199,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680"},{"bytes":272,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec"},{"bytes":380,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"}],"euid":0,"gate":{"application_candidate_count":40,"ready_for_remote_transaction_design":true,"reference_file_count":1,"unit_count":16},"health_input_files":[{"bytes":380,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"},{"bytes":128,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac"},{"bytes":1535,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b"}],"hostname":"cluster-admin","indexes":[{"bytes":10353,"path":"/var/www/homelab-cluster-admin/index.html","root":"/var/www/homelab-cluster-admin","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a","title_lines":["<title>Homelab Cluster Admin</title>","<h1>Homelab Cluster Admin</h1>","<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>","<div class=<REDACTED>","<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>","<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>","<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer","pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"]}],"listeners":{"rc":0,"stderr":"","stdout":"State  Recv-Q Send-Q Local Address:Port Peer Address:PortProcess                                   \nLISTEN 0      5            0.0.0.0:8080      0.0.0.0:*    users:((\"python3\",pid=439,fd=3))         \nLISTEN 0      2048         0.0.0.0:8081      0.0.0.0:*    users:((\"python3\",pid=487,fd=6))         \nLISTEN 0      244        127.0.0.1:5432      0.0.0.0:*    users:((\"postgres\",pid=469,fd=6))        \nLISTEN 0      4096   127.0.0.53%lo:53        0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=18))\nLISTEN 0      4096         0.0.0.0:5355      0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=12))\nLISTEN 0      4096      127.0.0.54:53        0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=20))\nLISTEN 0      128          0.0.0.0:22        0.0.0.0:*    users:((\"sshd\",pid=457,fd=3))            \nLISTEN 0      244            [::1]:5432         [::]:*    users:((\"postgres\",pid=469,fd=5))        \nLISTEN 0      4096            [::]:5355         [::]:*    users:((\"systemd-resolve\",pid=380,fd=14))\nLISTEN 0      128             [::]:22           [::]:*    users:((\"sshd\",pid=457,fd=4))            "},"matching_files":[{"bytes":133,"matches":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},{"bytes":149,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},{"bytes":154,"matches":[{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},{"bytes":151,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},{"bytes":231,"matches":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},{"bytes":154,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},{"bytes":138,"matches":[{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},{"bytes":127,"matches":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},{"bytes":616,"matches":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},{"bytes":1190,"matches":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},{"bytes":223,"matches":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},{"bytes":803,"matches":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},{"bytes":131,"matches":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},{"bytes":301,"matches":[{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},{"bytes":89,"matches":[{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f"},{"bytes":8444,"matches":[{"line":4,"text":"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')"},{"line":64,"text":" return f\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\n\""},{"line":82,"text":" old=<REDACTED>"},{"line":97,"text":"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)"},{"line":99,"text":"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))"}],"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a"},{"bytes":28599,"matches":[{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE =<REDACTED>"},{"line":14,"text":"COOKIE =<REDACTED>"},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":20,"text":"    return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":73,"text":"    return f\"\"\"<!doctype html><html lang=<REDACTED>"},{"line":152,"text":"    return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":220,"text":"    page = f\"\"\"<!doctype html><html lang=\"ru\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta http-equiv=\"refresh\" content=\"30\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\"post\" action=\"/logout\"><input type=\"hidden\" name=\"csrf\" value=\"{csrf}\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\"\"\""},{"line":243,"text":"        \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":"        \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":"        \"cluster_admin_up 1\","},{"line":246,"text":"        \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":"        \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":"        \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":"        \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":"        \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":"        \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":"        \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":"        \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":"        \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":"        \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":"        \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":"        \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":"        \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":"        \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":"        \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":"        \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":"        \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":"        \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":267,"text":"        lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6"},{"bytes":922,"matches":[{"line":2,"text":"first = getpass.getpass(\"Новый пароль Cluster Admin: \")"},{"line":12,"text":"directory = \"/etc/cluster-admin-incident-engine\""},{"line":16,"text":"account = pwd.getpwnam(\"clusteradmin\")"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3"},{"bytes":10640,"matches":[{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":36,"text":"    data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":"    connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":41,"text":"        cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":50,"text":"            if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":57,"text":"        cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":108,"text":"    line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe"},{"bytes":1857,"matches":[{"line":3,"text":"  \"type\": \"cluster-admin-stage1-foundation-migration-plan\","},{"line":43,"text":"    \"function\": \"cluster_admin_enrich_collector_run\","}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670"},{"bytes":1342,"matches":[{"line":3,"text":"  \"collector_call\": \"SELECT cluster_admin_sync_foundation(%s)\","},{"line":15,"text":"  \"function\": \"cluster_admin_sync_foundation\","},{"line":16,"text":"  \"function_signature\": \"cluster_admin_sync_foundation(bigint)\","},{"line":26,"text":"  \"type\": \"cluster-admin-stage2-migration-plan\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c"},{"bytes":2355,"matches":[{"line":51,"text":"  \"type\": \"cluster-admin-stage2-dynamic-acceptance-contract\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588"},{"bytes":2832,"matches":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},{"bytes":3471,"matches":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},{"bytes":1433,"matches":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},{"bytes":848,"matches":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":10353,"matches":[{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a"},{"bytes":168,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify"}],"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729"},{"bytes":259,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741"},{"bytes":134,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517"},{"bytes":141,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6"},{"bytes":189,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941"},{"bytes":131,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0"}],"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":199,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680"},{"bytes":272,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec"},{"bytes":380,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":1535,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b"},{"bytes":128,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac"},{"bytes":217,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"}],"process_lines":["    439       1 root     root     python3         /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin","    487       1 cluster+ cluster+ python3         /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1"],"proxies":[],"pve01_pushed_reference_files":[{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"}],"scan":{"directories_scanned":68,"index_count":1,"match_count":42},"units":[{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","Group":"clusteradmin","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","Group":"","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","Group":"root","LoadState":"loaded","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","User":"root","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}],"webroots":[]},"remote_target":{"evidence":[{"line":21,"target_count":1,"text":"scp -q -o BatchMode=yes -o ConnectTimeout=8 \"$TMP\"/*.txt debian@<PRIVATE_IPV4>:/tmp/"},{"line":24,"target_count":1,"text":"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \"$f\" && sudo install -m 0644 \"$f\" /var/lib/homelab-health/pve01-pushed/; done'"}],"target_cardinality":1,"target_host_is_private_ipv4":true,"target_host_sha256":"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69","target_user":"debian"},"rollback_restored":false,"rollback_started":false,"schema":15,"status":"OK","transaction_gate":{"application_architecture_found":true,"next_required_action":"build one corrected local+remote atomic transaction","pushed_health_reference_found":true,"ready_for_corrected_atomic_transaction":true,"remote_probe_root":true},"transport":{"attempts":[{"privilege":"sudo-root","rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":82754,"stdout_sha256":"603f5eb85c971d0bc029287cf91d9d0239b132dd409df5bbf8c5f47a7bb32738"}],"privilege":"sudo-root","success":true},"v14_exact_result":{"changes_made":false,"control_plane_error":null,"identity_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":10768,"output_sha256":"cc30d7f4e6be0ab5e78545d4537a39178b59e9f1a79227efaec134b78f6fc233","rollback_restored":null,"rollback_started":false,"runner_document_rc":0,"runner_json_sha256":"c932bbad4dd7f176e29621f3c549e2571baf7f84500ef2c5408a04c6a84083ff","runner_rc":0,"runner_status":"OK","runner_text_sha256":"70796c2456365c9224dd6ff02897d29412349eb1a5fea03c89ae9212217a03f8","runner_wrapper_matches_document":true,"state_document_sha256":"f3db8a869123d1266f906fdd02ca18a890141c2bd63e2bc57a3a0e7c5962f319","state_path_sha256":"2c050c8423a48aab8f448c94123e67f132381be3232e1d10274f4e87e1013b16","state_status":"OK"}}
 
 OUTPUT_END
 CHAT_OUTPUT_END