homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity
Files
9380ac8153f85bf2779ae3b48c3faadef69a6198
homelab-public-context/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z.json
T

39 lines
92 KiB
JSON
Raw Blame History

{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "verify",
"component": "homelab-cluster-admin-update-visibility",
"started_at_utc": "2026-08-05T12:55:27Z",
"finished_at_utc": "2026-08-05T12:55:29Z",
"reference_register_checked": true,
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
"error_register_checked": true,
"error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
"command_sha256": "4845b4e99e60efaa533b2a551d4bf1773085ce96ef0e4ce9737b920907b0553f",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
"change_declared": false,
"result_contract_valid": true,
"result_contract_status": null,
"result_contract_error": null,
"command_rc": 0,
"changes_made": false,
"rollback_started": false,
"rollback_restored": null,
"mutation_outcome": "NO_MUTATION",
"sanitized": true,
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "b672b2fde18de662e73d8a98d4cc036895f9de2007679edc3c085392050e66e6",
"sanitized_output_sha256": "c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z\",\"command_rc\":0,\"control_plane_error\":null,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_corrected_atomic_transaction\":true,\"remote_application_candidate_count\":40,\"remote_hostname\":\"cluster-admin\",\"remote_index_count\":1,\"remote_probe_privilege\":\"sudo-root\",\"remote_reference_file_count\":1,\"remote_target_cardinality\":1,\"remote_target_host_sha256\":\"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69\",\"remote_unit_count\":16,\"remote_webroot_config_count\":0,\"report_bytes\":85056,\"report_sha256\":\"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"v14_exact_result_identity_confirmed\":true,\"version\":1,\"webpanel_script_sha256\":\"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276\"}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z\",\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"remote_architecture\":{\"application_candidates\":[{\"bytes\":133,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},{\"bytes\":149,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},{\"bytes\":151,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},{\"bytes\":231,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},{\"bytes\":138,\"matches\":[{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},{\"bytes\":127,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},{\"bytes\":616,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},{\"bytes\":1190,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},{\"bytes\":223,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},{\"bytes\":803,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},{\"bytes\":131,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},{\"bytes\":301,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},{\"bytes\":89,\"matches\":[{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\"},{\"bytes\":8444,\"matches\":[{\"line\":4,\"text\":\"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')\"},{\"line\":64,\"text\":\" return f\\\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\\\n\\\"\"},{\"line\":82,\"text\":\" old=<REDACTED>\"},{\"line\":97,\"text\":\" p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)\"},{\"line\":99,\"text\":\" STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\"},{\"bytes\":28599,\"matches\":[{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =<REDACTED>\"},{\"line\":14,\"text\":\"COOKIE =<REDACTED>\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":20,\"text\":\" return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":73,\"text\":\" return f\\\"\\\"\\\"<!doctype html><html lang=<REDACTED>\"},{\"line\":152,\"text\":\" return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":220,\"text\":\" page = f\\\"\\\"\\\"<!doctype html><html lang=\\\"ru\\\"><head><meta charset=\\\"utf-8\\\"><meta name=\\\"viewport\\\" content=\\\"width=device-width,initial-scale=1\\\"><meta http-equiv=\\\"refresh\\\" content=\\\"30\\\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\\\"post\\\" action=\\\"/logout\\\"><input type=\\\"hidden\\\" name=\\\"csrf\\\" value=\\\"{csrf}\\\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\\\"\\\"\\\"\"},{\"line\":243,\"text\":\" \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\" \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\" \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\" \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\" \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\" \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\" \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\" \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\" \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\" \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\" \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\" \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\" \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\" \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\" \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\" \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\" \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\" \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\" \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\" \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\" \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":267,\"text\":\" lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\"},{\"bytes\":922,\"matches\":[{\"line\":2,\"text\":\"first = getpass.getpass(\\\"Новый пароль Cluster Admin: \\\")\"},{\"line\":12,\"text\":\"directory = \\\"/etc/cluster-admin-incident-engine\\\"\"},{\"line\":16,\"text\":\"account = pwd.getpwnam(\\\"clusteradmin\\\")\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\"},{\"bytes\":10640,\"matches\":[{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":36,\"text\":\" data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\" connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":41,\"text\":\" cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":50,\"text\":\" if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":57,\"text\":\" cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":108,\"text\":\" line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\"},{\"bytes\":1857,\"matches\":[{\"line\":3,\"text\":\" \\\"type\\\": \\\"cluster-admin-stage1-foundation-migration-plan\\\",\"},{\"line\":43,\"text\":\" \\\"function\\\": \\\"cluster_admin_enrich_collector_run\\\",\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\"},{\"bytes\":1342,\"matches\":[{\"line\":3,\"text\":\" \\\"collector_call\\\": \\\"SELECT cluster_admin_sync_foundation(%s)\\\",\"},{\"line\":15,\"text\":\" \\\"function\\\": \\\"cluster_admin_sync_foundation\\\",\"},{\"line\":16,\"text\":\" \\\"function_signature\\\": \\\"cluster_admin_sync_foundation(bigint)\\\",\"},{\"line\":26,\"text\":\" \\\"type\\\": \\\"cluster-admin-stage2-migration-plan\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\"},{\"bytes\":2355,\"matches\":[{\"line\":51,\"text\":\" \\\"type\\\": \\\"cluster-admin-stage2-dynamic-acceptance-contract\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\"},{\"bytes\":2832,\"matches\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\" line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\" echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},{\"bytes\":3471,\"matches\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\" echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\" echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},{\"bytes\":1433,\"matches\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},{\"bytes\":848,\"matches\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":10353,\"matches\":[{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\"},{\"bytes\":168,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729\"},{\"bytes\":259,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741\"},{\"bytes\":134,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517\"},{\"bytes\":141,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\"},{\"bytes\":189,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\"},{\"bytes\":131,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":199,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\"},{\"bytes\":272,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec\"},{\"bytes\":380,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"}],\"euid\":0,\"gate\":{\"application_candidate_count\":40,\"ready_for_remote_transaction_design\":true,\"reference_file_count\":1,\"unit_count\":16},\"health_input_files\":[{\"bytes\":380,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"},{\"bytes\":128,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\"},{\"bytes\":1535,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\"}],\"hostname\":\"cluster-admin\",\"indexes\":[{\"bytes\":10353,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"root\":\"/var/www/homelab-cluster-admin\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\",\"title_lines\":[\"<title>Homelab Cluster Admin</title>\",\"<h1>Homelab Cluster Admin</h1>\",\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\",\"<div class=<REDACTED>\",\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\",\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"]}],\"listeners\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"State Recv-Q Send-Q Local Address:Port Peer Address:PortProcess \\nLISTEN 0 5 0.0.0.0:8080 0.0.0.0:* users:((\\\"python3\\\",pid=439,fd=3)) \\nLISTEN 0 2048 0.0.0.0:8081 0.0.0.0:* users:((\\\"python3\\\",pid=487,fd=6)) \\nLISTEN 0 244 127.0.0.1:5432 0.0.0.0:* users:((\\\"postgres\\\",pid=469,fd=6)) \\nLISTEN 0 4096 127.0.0.53%lo:53 0.0.0.0:* users:((\\\"systemd-resolve\\\",pid=380,fd=18))\\nLISTEN 0 4096 0.0.0.0:5355 0.0.0.0:* users:((\\\"systemd-resolve\\\",pid=380,fd=12))\\nLISTEN 0 4096 127.0.0.54:53 0.0.0.0:* users:((\\\"systemd-resolve\\\",pid=380,fd=20))\\nLISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:((\\\"sshd\\\",pid=457,fd=3)) \\nLISTEN 0 244 [::1]:5432 [::]:* users:((\\\"postgres\\\",pid=469,fd=5)) \\nLISTEN 0 4096 [::]:5355 [::]:* users:((\\\"systemd-resolve\\\",pid=380,fd=14))\\nLISTEN 0 128 [::]:22 [::]:* users:((\\\"sshd\\\",pid=457,fd=4)) \"},\"matching_files\":[{\"bytes\":133,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},{\"bytes\":149,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},{\"bytes\":151,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},{\"bytes\":231,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},{\"bytes\":138,\"matches\":[{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},{\"bytes\":127,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},{\"bytes\":616,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},{\"bytes\":1190,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},{\"bytes\":223,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},{\"bytes\":803,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},{\"bytes\":131,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},{\"bytes\":301,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},{\"bytes\":89,\"matches\":[{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\"},{\"bytes\":8444,\"matches\":[{\"line\":4,\"text\":\"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')\"},{\"line\":64,\"text\":\" return f\\\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\\\n\\\"\"},{\"line\":82,\"text\":\" old=<REDACTED>\"},{\"line\":97,\"text\":\" p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)\"},{\"line\":99,\"text\":\" STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\"},{\"bytes\":28599,\"matches\":[{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =<REDACTED>\"},{\"line\":14,\"text\":\"COOKIE =<REDACTED>\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":20,\"text\":\" return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":73,\"text\":\" return f\\\"\\\"\\\"<!doctype html><html lang=<REDACTED>\"},{\"line\":152,\"text\":\" return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":220,\"text\":\" page = f\\\"\\\"\\\"<!doctype html><html lang=\\\"ru\\\"><head><meta charset=\\\"utf-8\\\"><meta name=\\\"viewport\\\" content=\\\"width=device-width,initial-scale=1\\\"><meta http-equiv=\\\"refresh\\\" content=\\\"30\\\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\\\"post\\\" action=\\\"/logout\\\"><input type=\\\"hidden\\\" name=\\\"csrf\\\" value=\\\"{csrf}\\\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\\\"\\\"\\\"\"},{\"line\":243,\"text\":\" \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\" \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\" \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\" \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\" \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\" \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\" \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\" \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\" \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\" \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\" \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\" \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\" \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\" \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\" \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\" \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\" \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\" \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\" \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\" \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\" \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":267,\"text\":\" lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\"},{\"bytes\":922,\"matches\":[{\"line\":2,\"text\":\"first = getpass.getpass(\\\"Новый пароль Cluster Admin: \\\")\"},{\"line\":12,\"text\":\"directory = \\\"/etc/cluster-admin-incident-engine\\\"\"},{\"line\":16,\"text\":\"account = pwd.getpwnam(\\\"clusteradmin\\\")\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\"},{\"bytes\":10640,\"matches\":[{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":36,\"text\":\" data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\" connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":41,\"text\":\" cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":50,\"text\":\" if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":57,\"text\":\" cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":108,\"text\":\" line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\"},{\"bytes\":1857,\"matches\":[{\"line\":3,\"text\":\" \\\"type\\\": \\\"cluster-admin-stage1-foundation-migration-plan\\\",\"},{\"line\":43,\"text\":\" \\\"function\\\": \\\"cluster_admin_enrich_collector_run\\\",\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\"},{\"bytes\":1342,\"matches\":[{\"line\":3,\"text\":\" \\\"collector_call\\\": \\\"SELECT cluster_admin_sync_foundation(%s)\\\",\"},{\"line\":15,\"text\":\" \\\"function\\\": \\\"cluster_admin_sync_foundation\\\",\"},{\"line\":16,\"text\":\" \\\"function_signature\\\": \\\"cluster_admin_sync_foundation(bigint)\\\",\"},{\"line\":26,\"text\":\" \\\"type\\\": \\\"cluster-admin-stage2-migration-plan\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\"},{\"bytes\":2355,\"matches\":[{\"line\":51,\"text\":\" \\\"type\\\": \\\"cluster-admin-stage2-dynamic-acceptance-contract\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\"},{\"bytes\":2832,\"matches\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\" line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\" echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},{\"bytes\":3471,\"matches\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\" echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\" echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},{\"bytes\":1433,\"matches\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},{\"bytes\":848,\"matches\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":10353,\"matches\":[{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\"},{\"bytes\":168,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729\"},{\"bytes\":259,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741\"},{\"bytes\":134,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517\"},{\"bytes\":141,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\"},{\"bytes\":189,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\"},{\"bytes\":131,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":199,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\"},{\"bytes\":272,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec\"},{\"bytes\":380,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":1535,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\"},{\"bytes\":128,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\"},{\"bytes\":217,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"}],\"process_lines\":[\" 439 1 root root python3 /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\",\" 487 1 cluster+ cluster+ python3 /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\"],\"proxies\":[],\"pve01_pushed_reference_files\":[{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"}],\"scan\":{\"directories_scanned\":68,\"index_count\":1,\"match_count\":42},\"units\":[{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"Group\":\"root\",\"LoadState\":\"loaded\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"root\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}],\"webroots\":[]},\"remote_target\":{\"evidence\":[{\"line\":21,\"target_count\":1,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$TMP\\\"/*.txt debian@<PRIVATE_IPV4>:/tmp/\"},{\"line\":24,\"target_count\":1,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@<PRIVATE_IPV4> 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \\\"$f\\\" && sudo install -m 0644 \\\"$f\\\" /var/lib/homelab-health/pve01-pushed/; done'\"}],\"target_cardinality\":1,\"target_host_is_private_ipv4\":true,\"target_host_sha256\":\"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69\",\"target_user\":\"debian\"},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":15,\"status\":\"OK\",\"transaction_gate\":{\"application_architecture_found\":true,\"next_required_action\":\"build one corrected local+remote atomic transaction\",\"pushed_health_reference_found\":true,\"ready_for_corrected_atomic_transaction\":true,\"remote_probe_root\":true},\"transport\":{\"attempts\":[{\"privilege\":\"sudo-root\",\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":82754,\"stdout_sha256\":\"603f5eb85c971d0bc029287cf91d9d0239b132dd409df5bbf8c5f47a7bb32738\"}],\"privilege\":\"sudo-root\",\"success\":true},\"v14_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":10768,\"output_sha256\":\"cc30d7f4e6be0ab5e78545d4537a39178b59e9f1a79227efaec134b78f6fc233\",\"rollback_restored\":null,\"rollback_started\":false,\"runner_document_rc\":0,\"runner_json_sha256\":\"c932bbad4dd7f176e29621f3c549e2571baf7f84500ef2c5408a04c6a84083ff\",\"runner_rc\":0,\"runner_status\":\"OK\",\"runner_text_sha256\":\"70796c2456365c9224dd6ff02897d29412349eb1a5fea03c89ae9212217a03f8\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"f3db8a869123d1266f906fdd02ca18a890141c2bd63e2bc57a3a0e7c5962f319\",\"state_path_sha256\":\"2c050c8423a48aab8f448c94123e67f132381be3232e1d10274f4e87e1013b16\",\"state_status\":\"OK\"}}\n"
}
View Git Blame Copy Permalink