From 212fedbb8ef4365eca4b2b94ae4b94ea39f253a3 Mon Sep 17 00:00:00 2001
From: homelab-runtime-publisher <homelab-runtime@localhost>
Date: Wed, 5 Aug 2026 14:36:18 +0000
Subject: [PATCH] runtime: publish
 HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z

---
 ...X-PRODUCER-READ-ONLY-20260805T143000Z.json | 38 +++++++++++++++++++
 ...EX-PRODUCER-READ-ONLY-20260805T143000Z.txt | 34 +++++++++++++++++
 runtime/latest.json                           | 22 +++++------
 runtime/latest.txt                            | 20 +++++-----
 4 files changed, 93 insertions(+), 21 deletions(-)
 create mode 100644 runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z.json
 create mode 100644 runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z.txt

diff --git a/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z.json b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z.json
new file mode 100644
index 0000000..6ee01a3
--- /dev/null
+++ b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z.json
@@ -0,0 +1,38 @@
+{
+  "schema_version": 1,
+  "channel": "homelab-runtime",
+  "command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z",
+  "status": "OK",
+  "rc": 0,
+  "host": "pve01",
+  "mode": "verify",
+  "component": "homelab-cluster-admin-update-visibility",
+  "started_at_utc": "2026-08-05T14:36:11Z",
+  "finished_at_utc": "2026-08-05T14:36:17Z",
+  "reference_register_checked": true,
+  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
+  "error_register_checked": true,
+  "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
+  "command_sha256": "d8613dc60bb222bd08ec67d6351af6f25843e63459cfa7dd0b0e0efb647b18ff",
+  "duplicate_failed_command_blocked": false,
+  "block_reason": null,
+  "execution_started": true,
+  "change_declared": false,
+  "result_contract_valid": true,
+  "result_contract_status": null,
+  "result_contract_error": null,
+  "command_rc": 0,
+  "changes_made": false,
+  "rollback_started": false,
+  "rollback_restored": null,
+  "mutation_outcome": "NO_MUTATION",
+  "sanitized": true,
+  "secrets_included": false,
+  "private_addresses_included": false,
+  "raw_evidence_retained_locally": true,
+  "raw_evidence_sha256": "7d4a33d0578aab214f4377ead7188d851c0c23e436a92a798b62ffd63eff3e8e",
+  "sanitized_output_sha256": "48cefb450e5f72ed68ed1bf09bbbc4015c82244cd46447d02720a5f3c822ee45",
+  "output_truncated_in_json": false,
+  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
+  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z\",\"command_rc\":0,\"control_plane_error\":null,\"exact_unit_count\":1,\"final_strategy\":\"live-baseline-late-read-with-atomic-recheck\",\"index_sha256_first\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"index_sha256_second\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"index_stable_over_3_seconds\":true,\"matching_source_count\":4,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_final_transaction\":true,\"remote_probe_root\":true,\"report_bytes\":27858,\"report_sha256\":\"76ff00a1e7d991d514a37a438f5107afb49f2713780f5fa33f25860bc8971e93\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"strong_candidate_count\":0,\"unique_producer_found\":false,\"unique_producer_path\":null,\"unique_producer_sha256\":null,\"v23_exact_result_identity_confirmed\":true,\"v23_exact_sha_drift_confirmed\":true,\"version\":1}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z\",\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"remote_producer_analysis\":{\"candidate_units\":[\"homelab-cluster-admin-webpanel-render.service\",\"homelab-cluster-admin-webpanel.service\"],\"euid\":0,\"exact_units\":[{\"cat_rc\":0,\"directory_reference\":true,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"}],\"gate\":{\"next_required_action\":\"use live-baseline same-transaction patch with regeneration guard\",\"ready_for_generator_patch_design\":false,\"strong_candidate_count\":0,\"unique_producer_source\":null},\"hostname\":\"cluster-admin\",\"index\":{\"bytes\":10353,\"ctime_ns\":1785940538091660296,\"gid\":0,\"inode\":135903,\"mode\":\"0o644\",\"mtime_ns\":1785940538091660296,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256_first\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"sha256_second\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"stable_over_3_seconds\":true,\"title_lines\":[\"<title>Homelab Cluster Admin</title>\",\"<h1>Homelab Cluster Admin</h1>\",\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\",\"<div class=<REDACTED>\",\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\",\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:30:59Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:00Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\",\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:31:02Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\",\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:05Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\",\"OK severity=<REDACTED>\",\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"],\"uid\":0},\"journal\":{\"homelab-cluster-admin-webpanel-render.service\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"Aug 05 14:33:28 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\\nAug 05 14:33:28 cluster-admin render-webpanel-cycle-break.sh[2018935]: STATUS=WARN TS=2026-08-05T14:33:28Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\nAug 05 14:33:28 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\\nAug 05 14:33:28 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\\nAug 05 14:34:37 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\\nAug 05 14:34:37 cluster-admin render-webpanel-cycle-break.sh[2019068]: STATUS=WARN TS=2026-08-05T14:34:37Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\nAug 05 14:34:37 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\\nAug 05 14:34:37 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\\nAug 05 14:35:38 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\\nAug 05 14:35:38 cluster-admin render-webpanel-cycle-break.sh[2019204]: STATUS=WARN TS=2026-08-05T14:35:38Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\nAug 05 14:35:38 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\\nAug 05 14:35:38 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\\n\"},\"homelab-cluster-admin-webpanel.service\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"Aug 05 14:33:07 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:33:07] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:04 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:04] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:08 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:08] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:23 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:23] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:38 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:38] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\n\"}},\"matches\":[{\"bytes\":1433,\"contexts\":[{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"score\":80,\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\",\"write_logic\":true},{\"bytes\":3708,\"contexts\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":39,\"text\":\"cat > \\\"$WEB/index.html\\\" <<HTML\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"score\":80,\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"write_logic\":true},{\"bytes\":301,\"contexts\":[{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"score\":50,\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"write_logic\":false},{\"bytes\":301,\"contexts\":[{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"score\":50,\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"write_logic\":false}],\"scan\":{\"directories_scanned\":65,\"files_scanned\":387,\"matching_files\":4},\"units\":[{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019159 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:35:38 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:47 UTC] ; stop_time=[Wed 2026-08-05 14:35:48 UTC] ; pid=2019217 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:35:47 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:14 UTC] ; pid=2019263 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:36:08 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019264 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:36:08 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019265 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:36:08 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019160 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:35:38 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":true,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:26:28 UTC] ; stop_time=[Wed 2026-08-05 14:26:28 UTC] ; pid=2017794 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"LoadState\":\"loaded\",\"Result\":\"exit-code\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:26:28 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}]},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":24,\"status\":\"OK\",\"transaction_gate\":{\"index_stable_over_sample\":true,\"ready_for_final_transaction\":true,\"strategy\":\"live-baseline-late-read-with-atomic-recheck\",\"strong_candidate_count\":0,\"unique_producer_found\":false,\"unique_producer_path\":null,\"unique_producer_sha256\":null},\"transport\":{\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":26124,\"stdout_sha256\":\"9694e9ab7966de769d9439dcd24d21cacb20a8a3a548c48d85544aad645d63f6\"},\"v23_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"exact_error\":\"remote python failed: Traceback (most recent call last):\\n  File \\\"<stdin>\\\", line 14, in <module>\\nRuntimeError: sha drift /var/www/homelab-cluster-admin/index.html\\n\",\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":843,\"output_sha256\":\"1bfd49a2a2f0b5f0ed1e2b386c132b63c6fcd7b7bb17b50e301783e803d0bc0c\",\"output_source\":\"runner_json\",\"rollback_restored\":false,\"rollback_started\":false,\"runner_document_rc\":1,\"runner_json_sha256\":\"eabc1c56bfd307512e2914d2163ab7e8accd5ea56adec9d2ae804e305c1c6fbe\",\"runner_rc\":1,\"runner_status\":\"FAIL\",\"runner_text_sha256\":\"677c6def8ad8cff8d01cec0c743d09978c1924f962fb9cc931472cad6f3d038a\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"16c1be27a9ba7efa83b40e4977f2b3cf83e6150dd809febc8a452662ed20dc20\",\"state_path_sha256\":\"2d957c8685cec0699340791b7bea723cb0cfd552a707eac454ec2d8aac23b87e\",\"state_status\":\"FAIL\"}}\n"
+}
diff --git a/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z.txt b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z.txt
new file mode 100644
index 0000000..6f483e9
--- /dev/null
+++ b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z.txt
@@ -0,0 +1,34 @@
+CHAT_OUTPUT_BEGIN
+COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z
+STATUS=OK
+RC=0
+HOST=pve01
+MODE=verify
+COMPONENT=homelab-cluster-admin-update-visibility
+REFERENCE_REGISTER_CHECK=OK
+REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
+ERROR_REGISTER_CHECK=OK
+ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
+COMMAND_SHA256=d8613dc60bb222bd08ec67d6351af6f25843e63459cfa7dd0b0e0efb647b18ff
+DUPLICATE_FAILED_COMMAND_BLOCKED=false
+EXECUTION_STARTED=true
+CHANGE_DECLARED=false
+RESULT_CONTRACT_VALID=true
+RESULT_CONTRACT_STATUS=NOT_APPLICABLE
+RESULT_CONTRACT_ERROR=NONE
+COMMAND_RC=0
+CHANGES_MADE=false
+ROLLBACK_STARTED=false
+ROLLBACK_RESTORED=null
+MUTATION_OUTCOME=NO_MUTATION
+SANITIZED=yes
+SECRETS_INCLUDED=no
+PRIVATE_ADDRESSES_INCLUDED=no
+RAW_EVIDENCE_SHA256=7d4a33d0578aab214f4377ead7188d851c0c23e436a92a798b62ffd63eff3e8e
+SANITIZED_OUTPUT_SHA256=48cefb450e5f72ed68ed1bf09bbbc4015c82244cd46447d02720a5f3c822ee45
+OUTPUT_BEGIN
+HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z","command_rc":0,"control_plane_error":null,"exact_unit_count":1,"final_strategy":"live-baseline-late-read-with-atomic-recheck","index_sha256_first":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","index_sha256_second":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","index_stable_over_3_seconds":true,"matching_source_count":4,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_final_transaction":true,"remote_probe_root":true,"report_bytes":27858,"report_sha256":"76ff00a1e7d991d514a37a438f5107afb49f2713780f5fa33f25860bc8971e93","rollback_restored":false,"rollback_started":false,"status":"OK","strong_candidate_count":0,"unique_producer_found":false,"unique_producer_path":null,"unique_producer_sha256":null,"v23_exact_result_identity_confirmed":true,"v23_exact_sha_drift_confirmed":true,"version":1}
+{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z","mutation_outcome":"NO_MUTATION","read_only":true,"remote_producer_analysis":{"candidate_units":["homelab-cluster-admin-webpanel-render.service","homelab-cluster-admin-webpanel.service"],"euid":0,"exact_units":[{"cat_rc":0,"directory_reference":true,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"}],"gate":{"next_required_action":"use live-baseline same-transaction patch with regeneration guard","ready_for_generator_patch_design":false,"strong_candidate_count":0,"unique_producer_source":null},"hostname":"cluster-admin","index":{"bytes":10353,"ctime_ns":1785940538091660296,"gid":0,"inode":135903,"mode":"0o644","mtime_ns":1785940538091660296,"path":"/var/www/homelab-cluster-admin/index.html","sha256_first":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","sha256_second":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","stable_over_3_seconds":true,"title_lines":["<title>Homelab Cluster Admin</title>","<h1>Homelab Cluster Admin</h1>","<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>","<div class=<REDACTED>","<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>","<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>","<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer","pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:30:59Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:00Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2","pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:31:02Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0","edge ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:05Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active","OK severity=<REDACTED>","OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"],"uid":0},"journal":{"homelab-cluster-admin-webpanel-render.service":{"rc":0,"stderr":"","stdout":"Aug 05 14:33:28 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\nAug 05 14:33:28 cluster-admin render-webpanel-cycle-break.sh[2018935]: STATUS=WARN TS=2026-08-05T14:33:28Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\nAug 05 14:33:28 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\nAug 05 14:33:28 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\nAug 05 14:34:37 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\nAug 05 14:34:37 cluster-admin render-webpanel-cycle-break.sh[2019068]: STATUS=WARN TS=2026-08-05T14:34:37Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\nAug 05 14:34:37 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\nAug 05 14:34:37 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\nAug 05 14:35:38 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\nAug 05 14:35:38 cluster-admin render-webpanel-cycle-break.sh[2019204]: STATUS=WARN TS=2026-08-05T14:35:38Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\nAug 05 14:35:38 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\nAug 05 14:35:38 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\n"},"homelab-cluster-admin-webpanel.service":{"rc":0,"stderr":"","stdout":"Aug 05 14:33:07 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:33:07] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:04 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:04] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:08 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:08] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:23 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:23] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:38 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:38] \"HEAD /status.txt HTTP/1.1\" 200 -\n"}},"matches":[{"bytes":1433,"contexts":[{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"directory_path":true,"exact_path":false,"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","score":80,"sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792","write_logic":true},{"bytes":3708,"contexts":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":39,"text":"cat > \"$WEB/index.html\" <<HTML"}],"directory_path":true,"exact_path":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","score":80,"sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","write_logic":true},{"bytes":301,"contexts":[{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"directory_path":true,"exact_path":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","score":50,"sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","write_logic":false},{"bytes":301,"contexts":[{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"directory_path":true,"exact_path":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","score":50,"sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","write_logic":false}],"scan":{"directories_scanned":65,"files_scanned":387,"matching_files":4},"units":[{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019159 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LastTriggerUSec":"Wed 2026-08-05 14:35:38 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:47 UTC] ; stop_time=[Wed 2026-08-05 14:35:48 UTC] ; pid=2019217 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LastTriggerUSec":"Wed 2026-08-05 14:35:47 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:14 UTC] ; pid=2019263 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LastTriggerUSec":"Wed 2026-08-05 14:36:08 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019264 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LastTriggerUSec":"Wed 2026-08-05 14:36:08 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019265 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LastTriggerUSec":"Wed 2026-08-05 14:36:08 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019160 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LastTriggerUSec":"Wed 2026-08-05 14:35:38 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":true,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:26:28 UTC] ; stop_time=[Wed 2026-08-05 14:26:28 UTC] ; pid=2017794 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","LoadState":"loaded","Result":"exit-code","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LastTriggerUSec":"Wed 2026-08-05 14:26:28 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}]},"rollback_restored":false,"rollback_started":false,"schema":24,"status":"OK","transaction_gate":{"index_stable_over_sample":true,"ready_for_final_transaction":true,"strategy":"live-baseline-late-read-with-atomic-recheck","strong_candidate_count":0,"unique_producer_found":false,"unique_producer_path":null,"unique_producer_sha256":null},"transport":{"rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":26124,"stdout_sha256":"9694e9ab7966de769d9439dcd24d21cacb20a8a3a548c48d85544aad645d63f6"},"v23_exact_result":{"changes_made":false,"control_plane_error":null,"exact_error":"remote python failed: Traceback (most recent call last):\n  File \"<stdin>\", line 14, in <module>\nRuntimeError: sha drift /var/www/homelab-cluster-admin/index.html\n","identity_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":843,"output_sha256":"1bfd49a2a2f0b5f0ed1e2b386c132b63c6fcd7b7bb17b50e301783e803d0bc0c","output_source":"runner_json","rollback_restored":false,"rollback_started":false,"runner_document_rc":1,"runner_json_sha256":"eabc1c56bfd307512e2914d2163ab7e8accd5ea56adec9d2ae804e305c1c6fbe","runner_rc":1,"runner_status":"FAIL","runner_text_sha256":"677c6def8ad8cff8d01cec0c743d09978c1924f962fb9cc931472cad6f3d038a","runner_wrapper_matches_document":true,"state_document_sha256":"16c1be27a9ba7efa83b40e4977f2b3cf83e6150dd809febc8a452662ed20dc20","state_path_sha256":"2d957c8685cec0699340791b7bea723cb0cfd552a707eac454ec2d8aac23b87e","state_status":"FAIL"}}
+
+OUTPUT_END
+CHAT_OUTPUT_END
diff --git a/runtime/latest.json b/runtime/latest.json
index e4a6ab0..6ee01a3 100644
--- a/runtime/latest.json
+++ b/runtime/latest.json
@@ -1,19 +1,19 @@
 {
   "schema_version": 1,
   "channel": "homelab-runtime",
-  "command_id": "HOMELAB-CLUSTER-KB-V1-ERROR-REGISTRY-EXACT-SHAPE-DIAGNOSTIC-READ-ONLY-20260805T142600Z",
-  "status": "FAIL",
-  "rc": 1,
+  "command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z",
+  "status": "OK",
+  "rc": 0,
   "host": "pve01",
   "mode": "verify",
-  "component": "cluster-kb-error-v2",
-  "started_at_utc": "2026-08-05T14:33:39Z",
-  "finished_at_utc": "2026-08-05T14:33:39Z",
+  "component": "homelab-cluster-admin-update-visibility",
+  "started_at_utc": "2026-08-05T14:36:11Z",
+  "finished_at_utc": "2026-08-05T14:36:17Z",
   "reference_register_checked": true,
   "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
   "error_register_checked": true,
   "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
-  "command_sha256": "e564bc2017b40dcb9c1f0c1790dd7a29ad8c804cc8f019bd2d5725c0a80ee6df",
+  "command_sha256": "d8613dc60bb222bd08ec67d6351af6f25843e63459cfa7dd0b0e0efb647b18ff",
   "duplicate_failed_command_blocked": false,
   "block_reason": null,
   "execution_started": true,
@@ -21,7 +21,7 @@
   "result_contract_valid": true,
   "result_contract_status": null,
   "result_contract_error": null,
-  "command_rc": 1,
+  "command_rc": 0,
   "changes_made": false,
   "rollback_started": false,
   "rollback_restored": null,
@@ -30,9 +30,9 @@
   "secrets_included": false,
   "private_addresses_included": false,
   "raw_evidence_retained_locally": true,
-  "raw_evidence_sha256": "c3e7aaf84109fbd9c5a3533e3ca6cd0e2bcfb7975542872cbb2d561d7b4fd83b",
-  "sanitized_output_sha256": "c3e7aaf84109fbd9c5a3533e3ca6cd0e2bcfb7975542872cbb2d561d7b4fd83b",
+  "raw_evidence_sha256": "7d4a33d0578aab214f4377ead7188d851c0c23e436a92a798b62ffd63eff3e8e",
+  "sanitized_output_sha256": "48cefb450e5f72ed68ed1bf09bbbc4015c82244cd46447d02720a5f3c822ee45",
   "output_truncated_in_json": false,
   "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
-  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-KB-V1-ERROR-REGISTRY-EXACT-SHAPE-DIAGNOSTIC-READ-ONLY-20260805T142600Z\",\"command_rc\":1,\"mutation_outcome\":\"NO_MUTATION\",\"output_truncated_in_json\":false,\"rc\":1,\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"FAIL\",\"version\":1}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-KB-V1-ERROR-REGISTRY-EXACT-SHAPE-DIAGNOSTIC-READ-ONLY-20260805T142600Z\",\"command_rc\":1,\"detail\":{\"bundle_bytes_ok\":true,\"bundle_exists\":true,\"bundle_sha256_ok\":true,\"capsule_sha256_bound\":false,\"command_id_bound\":true,\"contract_exact\":true,\"identity_verified\":false,\"next_action_exact\":true,\"output_exact_or_embedded_count\":1,\"root_cause_exact\":true,\"runner_json_exact_count\":1,\"runner_text_exact_count\":1},\"mutation_outcome\":\"NO_MUTATION\",\"next_action\":\"RUN_ONE_READ_ONLY_PREVIOUS_EXACT_EVIDENCE_DIAGNOSTIC\",\"output_truncated_in_json\":false,\"rc\":1,\"read_only\":true,\"rollback_restored\":null,\"rollback_started\":false,\"root_cause_classification\":\"PREVIOUS_FAILED_MUTATION_IDENTITY_NOT_REVERIFIED\",\"schema\":1,\"status\":\"FAIL\"}\n"
+  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z\",\"command_rc\":0,\"control_plane_error\":null,\"exact_unit_count\":1,\"final_strategy\":\"live-baseline-late-read-with-atomic-recheck\",\"index_sha256_first\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"index_sha256_second\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"index_stable_over_3_seconds\":true,\"matching_source_count\":4,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_final_transaction\":true,\"remote_probe_root\":true,\"report_bytes\":27858,\"report_sha256\":\"76ff00a1e7d991d514a37a438f5107afb49f2713780f5fa33f25860bc8971e93\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"strong_candidate_count\":0,\"unique_producer_found\":false,\"unique_producer_path\":null,\"unique_producer_sha256\":null,\"v23_exact_result_identity_confirmed\":true,\"v23_exact_sha_drift_confirmed\":true,\"version\":1}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z\",\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"remote_producer_analysis\":{\"candidate_units\":[\"homelab-cluster-admin-webpanel-render.service\",\"homelab-cluster-admin-webpanel.service\"],\"euid\":0,\"exact_units\":[{\"cat_rc\":0,\"directory_reference\":true,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"}],\"gate\":{\"next_required_action\":\"use live-baseline same-transaction patch with regeneration guard\",\"ready_for_generator_patch_design\":false,\"strong_candidate_count\":0,\"unique_producer_source\":null},\"hostname\":\"cluster-admin\",\"index\":{\"bytes\":10353,\"ctime_ns\":1785940538091660296,\"gid\":0,\"inode\":135903,\"mode\":\"0o644\",\"mtime_ns\":1785940538091660296,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256_first\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"sha256_second\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"stable_over_3_seconds\":true,\"title_lines\":[\"<title>Homelab Cluster Admin</title>\",\"<h1>Homelab Cluster Admin</h1>\",\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\",\"<div class=<REDACTED>\",\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\",\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:30:59Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:00Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\",\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:31:02Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\",\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:05Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\",\"OK severity=<REDACTED>\",\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"],\"uid\":0},\"journal\":{\"homelab-cluster-admin-webpanel-render.service\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"Aug 05 14:33:28 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\\nAug 05 14:33:28 cluster-admin render-webpanel-cycle-break.sh[2018935]: STATUS=WARN TS=2026-08-05T14:33:28Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\nAug 05 14:33:28 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\\nAug 05 14:33:28 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\\nAug 05 14:34:37 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\\nAug 05 14:34:37 cluster-admin render-webpanel-cycle-break.sh[2019068]: STATUS=WARN TS=2026-08-05T14:34:37Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\nAug 05 14:34:37 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\\nAug 05 14:34:37 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\\nAug 05 14:35:38 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\\nAug 05 14:35:38 cluster-admin render-webpanel-cycle-break.sh[2019204]: STATUS=WARN TS=2026-08-05T14:35:38Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\nAug 05 14:35:38 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\\nAug 05 14:35:38 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\\n\"},\"homelab-cluster-admin-webpanel.service\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"Aug 05 14:33:07 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:33:07] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:04 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:04] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:08 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:08] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:23 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:23] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:38 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:38] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\n\"}},\"matches\":[{\"bytes\":1433,\"contexts\":[{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"score\":80,\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\",\"write_logic\":true},{\"bytes\":3708,\"contexts\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":39,\"text\":\"cat > \\\"$WEB/index.html\\\" <<HTML\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"score\":80,\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"write_logic\":true},{\"bytes\":301,\"contexts\":[{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"score\":50,\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"write_logic\":false},{\"bytes\":301,\"contexts\":[{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"score\":50,\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"write_logic\":false}],\"scan\":{\"directories_scanned\":65,\"files_scanned\":387,\"matching_files\":4},\"units\":[{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019159 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:35:38 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:47 UTC] ; stop_time=[Wed 2026-08-05 14:35:48 UTC] ; pid=2019217 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:35:47 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:14 UTC] ; pid=2019263 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:36:08 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019264 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:36:08 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019265 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:36:08 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019160 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:35:38 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":true,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:26:28 UTC] ; stop_time=[Wed 2026-08-05 14:26:28 UTC] ; pid=2017794 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"LoadState\":\"loaded\",\"Result\":\"exit-code\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:26:28 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}]},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":24,\"status\":\"OK\",\"transaction_gate\":{\"index_stable_over_sample\":true,\"ready_for_final_transaction\":true,\"strategy\":\"live-baseline-late-read-with-atomic-recheck\",\"strong_candidate_count\":0,\"unique_producer_found\":false,\"unique_producer_path\":null,\"unique_producer_sha256\":null},\"transport\":{\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":26124,\"stdout_sha256\":\"9694e9ab7966de769d9439dcd24d21cacb20a8a3a548c48d85544aad645d63f6\"},\"v23_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"exact_error\":\"remote python failed: Traceback (most recent call last):\\n  File \\\"<stdin>\\\", line 14, in <module>\\nRuntimeError: sha drift /var/www/homelab-cluster-admin/index.html\\n\",\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":843,\"output_sha256\":\"1bfd49a2a2f0b5f0ed1e2b386c132b63c6fcd7b7bb17b50e301783e803d0bc0c\",\"output_source\":\"runner_json\",\"rollback_restored\":false,\"rollback_started\":false,\"runner_document_rc\":1,\"runner_json_sha256\":\"eabc1c56bfd307512e2914d2163ab7e8accd5ea56adec9d2ae804e305c1c6fbe\",\"runner_rc\":1,\"runner_status\":\"FAIL\",\"runner_text_sha256\":\"677c6def8ad8cff8d01cec0c743d09978c1924f962fb9cc931472cad6f3d038a\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"16c1be27a9ba7efa83b40e4977f2b3cf83e6150dd809febc8a452662ed20dc20\",\"state_path_sha256\":\"2d957c8685cec0699340791b7bea723cb0cfd552a707eac454ec2d8aac23b87e\",\"state_status\":\"FAIL\"}}\n"
 }
diff --git a/runtime/latest.txt b/runtime/latest.txt
index dc2eb39..6f483e9 100644
--- a/runtime/latest.txt
+++ b/runtime/latest.txt
@@ -1,22 +1,22 @@
 CHAT_OUTPUT_BEGIN
-COMMAND_ID=HOMELAB-CLUSTER-KB-V1-ERROR-REGISTRY-EXACT-SHAPE-DIAGNOSTIC-READ-ONLY-20260805T142600Z
-STATUS=FAIL
-RC=1
+COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z
+STATUS=OK
+RC=0
 HOST=pve01
 MODE=verify
-COMPONENT=cluster-kb-error-v2
+COMPONENT=homelab-cluster-admin-update-visibility
 REFERENCE_REGISTER_CHECK=OK
 REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
 ERROR_REGISTER_CHECK=OK
 ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
-COMMAND_SHA256=e564bc2017b40dcb9c1f0c1790dd7a29ad8c804cc8f019bd2d5725c0a80ee6df
+COMMAND_SHA256=d8613dc60bb222bd08ec67d6351af6f25843e63459cfa7dd0b0e0efb647b18ff
 DUPLICATE_FAILED_COMMAND_BLOCKED=false
 EXECUTION_STARTED=true
 CHANGE_DECLARED=false
 RESULT_CONTRACT_VALID=true
 RESULT_CONTRACT_STATUS=NOT_APPLICABLE
 RESULT_CONTRACT_ERROR=NONE
-COMMAND_RC=1
+COMMAND_RC=0
 CHANGES_MADE=false
 ROLLBACK_STARTED=false
 ROLLBACK_RESTORED=null
@@ -24,11 +24,11 @@ MUTATION_OUTCOME=NO_MUTATION
 SANITIZED=yes
 SECRETS_INCLUDED=no
 PRIVATE_ADDRESSES_INCLUDED=no
-RAW_EVIDENCE_SHA256=c3e7aaf84109fbd9c5a3533e3ca6cd0e2bcfb7975542872cbb2d561d7b4fd83b
-SANITIZED_OUTPUT_SHA256=c3e7aaf84109fbd9c5a3533e3ca6cd0e2bcfb7975542872cbb2d561d7b4fd83b
+RAW_EVIDENCE_SHA256=7d4a33d0578aab214f4377ead7188d851c0c23e436a92a798b62ffd63eff3e8e
+SANITIZED_OUTPUT_SHA256=48cefb450e5f72ed68ed1bf09bbbc4015c82244cd46447d02720a5f3c822ee45
 OUTPUT_BEGIN
-HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-KB-V1-ERROR-REGISTRY-EXACT-SHAPE-DIAGNOSTIC-READ-ONLY-20260805T142600Z","command_rc":1,"mutation_outcome":"NO_MUTATION","output_truncated_in_json":false,"rc":1,"rollback_restored":null,"rollback_started":false,"status":"FAIL","version":1}
-{"changes_made":false,"command_id":"HOMELAB-CLUSTER-KB-V1-ERROR-REGISTRY-EXACT-SHAPE-DIAGNOSTIC-READ-ONLY-20260805T142600Z","command_rc":1,"detail":{"bundle_bytes_ok":true,"bundle_exists":true,"bundle_sha256_ok":true,"capsule_sha256_bound":false,"command_id_bound":true,"contract_exact":true,"identity_verified":false,"next_action_exact":true,"output_exact_or_embedded_count":1,"root_cause_exact":true,"runner_json_exact_count":1,"runner_text_exact_count":1},"mutation_outcome":"NO_MUTATION","next_action":"RUN_ONE_READ_ONLY_PREVIOUS_EXACT_EVIDENCE_DIAGNOSTIC","output_truncated_in_json":false,"rc":1,"read_only":true,"rollback_restored":null,"rollback_started":false,"root_cause_classification":"PREVIOUS_FAILED_MUTATION_IDENTITY_NOT_REVERIFIED","schema":1,"status":"FAIL"}
+HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z","command_rc":0,"control_plane_error":null,"exact_unit_count":1,"final_strategy":"live-baseline-late-read-with-atomic-recheck","index_sha256_first":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","index_sha256_second":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","index_stable_over_3_seconds":true,"matching_source_count":4,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_final_transaction":true,"remote_probe_root":true,"report_bytes":27858,"report_sha256":"76ff00a1e7d991d514a37a438f5107afb49f2713780f5fa33f25860bc8971e93","rollback_restored":false,"rollback_started":false,"status":"OK","strong_candidate_count":0,"unique_producer_found":false,"unique_producer_path":null,"unique_producer_sha256":null,"v23_exact_result_identity_confirmed":true,"v23_exact_sha_drift_confirmed":true,"version":1}
+{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z","mutation_outcome":"NO_MUTATION","read_only":true,"remote_producer_analysis":{"candidate_units":["homelab-cluster-admin-webpanel-render.service","homelab-cluster-admin-webpanel.service"],"euid":0,"exact_units":[{"cat_rc":0,"directory_reference":true,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"}],"gate":{"next_required_action":"use live-baseline same-transaction patch with regeneration guard","ready_for_generator_patch_design":false,"strong_candidate_count":0,"unique_producer_source":null},"hostname":"cluster-admin","index":{"bytes":10353,"ctime_ns":1785940538091660296,"gid":0,"inode":135903,"mode":"0o644","mtime_ns":1785940538091660296,"path":"/var/www/homelab-cluster-admin/index.html","sha256_first":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","sha256_second":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","stable_over_3_seconds":true,"title_lines":["<title>Homelab Cluster Admin</title>","<h1>Homelab Cluster Admin</h1>","<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>","<div class=<REDACTED>","<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>","<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>","<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer","pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:30:59Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:00Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2","pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:31:02Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0","edge ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:05Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active","OK severity=<REDACTED>","OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"],"uid":0},"journal":{"homelab-cluster-admin-webpanel-render.service":{"rc":0,"stderr":"","stdout":"Aug 05 14:33:28 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\nAug 05 14:33:28 cluster-admin render-webpanel-cycle-break.sh[2018935]: STATUS=WARN TS=2026-08-05T14:33:28Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\nAug 05 14:33:28 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\nAug 05 14:33:28 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\nAug 05 14:34:37 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\nAug 05 14:34:37 cluster-admin render-webpanel-cycle-break.sh[2019068]: STATUS=WARN TS=2026-08-05T14:34:37Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\nAug 05 14:34:37 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\nAug 05 14:34:37 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\nAug 05 14:35:38 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\nAug 05 14:35:38 cluster-admin render-webpanel-cycle-break.sh[2019204]: STATUS=WARN TS=2026-08-05T14:35:38Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\nAug 05 14:35:38 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\nAug 05 14:35:38 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\n"},"homelab-cluster-admin-webpanel.service":{"rc":0,"stderr":"","stdout":"Aug 05 14:33:07 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:33:07] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:04 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:04] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:08 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:08] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:23 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:23] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:38 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:38] \"HEAD /status.txt HTTP/1.1\" 200 -\n"}},"matches":[{"bytes":1433,"contexts":[{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"directory_path":true,"exact_path":false,"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","score":80,"sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792","write_logic":true},{"bytes":3708,"contexts":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":39,"text":"cat > \"$WEB/index.html\" <<HTML"}],"directory_path":true,"exact_path":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","score":80,"sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","write_logic":true},{"bytes":301,"contexts":[{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"directory_path":true,"exact_path":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","score":50,"sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","write_logic":false},{"bytes":301,"contexts":[{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"directory_path":true,"exact_path":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","score":50,"sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","write_logic":false}],"scan":{"directories_scanned":65,"files_scanned":387,"matching_files":4},"units":[{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019159 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LastTriggerUSec":"Wed 2026-08-05 14:35:38 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:47 UTC] ; stop_time=[Wed 2026-08-05 14:35:48 UTC] ; pid=2019217 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LastTriggerUSec":"Wed 2026-08-05 14:35:47 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:14 UTC] ; pid=2019263 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LastTriggerUSec":"Wed 2026-08-05 14:36:08 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019264 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LastTriggerUSec":"Wed 2026-08-05 14:36:08 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019265 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LastTriggerUSec":"Wed 2026-08-05 14:36:08 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019160 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LastTriggerUSec":"Wed 2026-08-05 14:35:38 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":true,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:26:28 UTC] ; stop_time=[Wed 2026-08-05 14:26:28 UTC] ; pid=2017794 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","LoadState":"loaded","Result":"exit-code","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LastTriggerUSec":"Wed 2026-08-05 14:26:28 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}]},"rollback_restored":false,"rollback_started":false,"schema":24,"status":"OK","transaction_gate":{"index_stable_over_sample":true,"ready_for_final_transaction":true,"strategy":"live-baseline-late-read-with-atomic-recheck","strong_candidate_count":0,"unique_producer_found":false,"unique_producer_path":null,"unique_producer_sha256":null},"transport":{"rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":26124,"stdout_sha256":"9694e9ab7966de769d9439dcd24d21cacb20a8a3a548c48d85544aad645d63f6"},"v23_exact_result":{"changes_made":false,"control_plane_error":null,"exact_error":"remote python failed: Traceback (most recent call last):\n  File \"<stdin>\", line 14, in <module>\nRuntimeError: sha drift /var/www/homelab-cluster-admin/index.html\n","identity_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":843,"output_sha256":"1bfd49a2a2f0b5f0ed1e2b386c132b63c6fcd7b7bb17b50e301783e803d0bc0c","output_source":"runner_json","rollback_restored":false,"rollback_started":false,"runner_document_rc":1,"runner_json_sha256":"eabc1c56bfd307512e2914d2163ab7e8accd5ea56adec9d2ae804e305c1c6fbe","runner_rc":1,"runner_status":"FAIL","runner_text_sha256":"677c6def8ad8cff8d01cec0c743d09978c1924f962fb9cc931472cad6f3d038a","runner_wrapper_matches_document":true,"state_document_sha256":"16c1be27a9ba7efa83b40e4977f2b3cf83e6150dd809febc8a452662ed20dc20","state_path_sha256":"2d957c8685cec0699340791b7bea723cb0cfd552a707eac454ec2d8aac23b87e","state_status":"FAIL"}}
 
 OUTPUT_END
 CHAT_OUTPUT_END