diff --git a/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z.json b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z.json
new file mode 100644
index 0000000..27c8a75
--- /dev/null
+++ b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z.json
@@ -0,0 +1,38 @@
+{
+  "schema_version": 1,
+  "channel": "homelab-runtime",
+  "command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z",
+  "status": "OK",
+  "rc": 0,
+  "host": "pve01",
+  "mode": "verify",
+  "component": "homelab-cluster-admin-update-visibility",
+  "started_at_utc": "2026-08-05T14:20:20Z",
+  "finished_at_utc": "2026-08-05T14:20:20Z",
+  "reference_register_checked": true,
+  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
+  "error_register_checked": true,
+  "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
+  "command_sha256": "b4bce2469c19f1a3186f956193115529e14e4df31c9798c2de3a08fa8dd1c9ef",
+  "duplicate_failed_command_blocked": false,
+  "block_reason": null,
+  "execution_started": true,
+  "change_declared": false,
+  "result_contract_valid": true,
+  "result_contract_status": null,
+  "result_contract_error": null,
+  "command_rc": 0,
+  "changes_made": false,
+  "rollback_started": false,
+  "rollback_restored": null,
+  "mutation_outcome": "NO_MUTATION",
+  "sanitized": true,
+  "secrets_included": false,
+  "private_addresses_included": false,
+  "raw_evidence_retained_locally": true,
+  "raw_evidence_sha256": "81f3a7b84d9eec86671f34f89bc59d99a95dd08ec9d6911e6ea399964e0e9718",
+  "sanitized_output_sha256": "691a9fb3ca3ff21163b447d98d7f8055f3e7ad70e86dafd2b73cdd5e9d118969",
+  "output_truncated_in_json": false,
+  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
+  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z\",\"command_rc\":0,\"control_plane_error\":null,\"index_changed_diff_lines\":110,\"index_current_sha256\":\"686e55e3934e22180a3889a0ee9ba8cdcbb92b5d9d1bc5418080f701a5796172\",\"index_drift\":true,\"index_expected_sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"index_path\":\"/var/www/homelab-cluster-admin/index.html\",\"managed_marker_present\":false,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_corrected_transaction\":true,\"reference_current_sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"reference_drift\":false,\"reference_expected_sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"remote_probe_root\":true,\"report_bytes\":33176,\"report_sha256\":\"04df393f3f5a7703399153a129a8458a3c9980f45c4cb00b1f9bd8b62182c15c\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"v19_exact_result_identity_confirmed\":true,\"v21_exact_result_identity_confirmed\":true,\"v21_exact_sha_drift_confirmed\":true,\"version\":1}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z\",\"current_files\":{\"index\":{\"backend_marker_begin\":false,\"backend_marker_end\":false,\"bytes\":10353,\"ctime_ns\":1785939615262660290,\"expected_sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"gid\":0,\"index_marker_begin\":false,\"index_marker_end\":false,\"inode\":135903,\"matches_v19\":false,\"mode\":420,\"mtime_ns\":1785939615262660290,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"686e55e3934e22180a3889a0ee9ba8cdcbb92b5d9d1bc5418080f701a5796172\",\"uid\":0},\"reference\":{\"backend_marker_begin\":false,\"backend_marker_end\":false,\"bytes\":3708,\"ctime_ns\":1783706500680296060,\"expected_sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"gid\":0,\"index_marker_begin\":false,\"index_marker_end\":false,\"inode\":135901,\"matches_v19\":true,\"mode\":493,\"mtime_ns\":1783706500680296060,\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"uid\":0}},\"drift_analysis\":{\"index_changed_diff_lines\":110,\"index_drift\":true,\"managed_marker_present\":false,\"next_required_action\":\"build corrected atomic transaction using current index SHA\",\"ready_for_corrected_transaction\":true,\"reference_changed_diff_lines\":24,\"reference_drift\":false},\"index_diff\":{\"lines\":[\"--- V19/index.html\",\"+++ CURRENT/index.html\",\"@@ -16,65 +16,65 @@\",\" </style>\",\" </head>\",\" <body>\",\" <h1>Homelab Cluster Admin</h1>\",\"-<div class=\\\"small\\\">generated UTC: 2026-08-05T13:20:48Z · VM180 · [PRIVATE_IP] · refresh 60s</div>\",\"+<div class=\\\"small\\\">generated UTC: 2026-08-05T14:20:15Z · VM180 · <PRIVATE_IPV4> · refresh 60s</div>\",\" <p><span class=\\\"badge\\\">REVIEW</span></p>\",\" \",\" <div class=\\\"grid\\\">\",\"-<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"+<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\" FAILED_UNITS=1\",\" QGA=active</pre></div>\",\"-<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"-<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"+<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"+<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\" <div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\",\" </div>\",\" \",\"-<div class=<REDACTED>\",\"+<div class=<REDACTED>\",\" <div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\",\"-<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\"+<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=<PRIVATE_IPV4> MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\" <div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\",\"-generated_utc=2026-08-05T13:19:08Z\",\"+generated_utc=2026-08-05T14:15:25Z\",\" policy=critical_failures_are_bad_optional_tcp_is_inventory_info\",\" \",\" ## Restricted probes\",\"-pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"-pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\",\"-pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\",\"-edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\",\"+pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"+pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T14:15:27Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\",\"+pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:15:29Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\",\"+edge ssh_rc=0 STATUS=OK TS=2026-08-05T14:15:31Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\",\" \",\" ## Service map checks\",\"-OK severity=<REDACTED>\",\"-OK severity=critical kind=tcp name=pve01_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=tcp name=pve01_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\",\"-OK severity=critical kind=tcp name=pve01_health target=[PRIVATE_IP] value=9101 tcp_9101_rc=0\",\"-OK severity=critical kind=tcp name=pve02_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=tcp name=pve02_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\",\"-OK severity=critical kind=tcp name=pve03_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=tcp name=pve03_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\",\"-OK severity=critical kind=tcp name=dns1_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0\",\"-OK severity=critical kind=tcp name=dns2_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0\",\"-OK severity=critical kind=tcp name=edge_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=ping name=vm130_edge_vm_ping target=[PRIVATE_IP] value=- ping_rc=0\",\"-OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=ping name=vm150_nextcloud_ping target=[PRIVATE_IP] value=- ping_rc=0\",\"-OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\",\"-INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\",\"-OK severity=critical kind=ping name=vm160_forum_prod_ping target=[PRIVATE_IP] value=- ping_rc=0\",\"-OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=0\",\"-INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\",\"-OK severity=critical kind=ping name=vm170_core_apps_ping target=[PRIVATE_IP] value=- ping_rc=0\",\"-OK severity=critical kind=tcp name=vm170_core_apps_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\",\"-INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\",\"-OK severity=critical kind=ping name=vm171_monitoring_ping target=[PRIVATE_IP] value=- ping_rc=0\",\"-OK severity=critical kind=tcp name=vm171_monitoring_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\",\"-INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\",\"+OK severity=<REDACTED>\",\"+OK severity=critical kind=tcp name=pve01_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=tcp name=pve01_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0\",\"+OK severity=critical kind=tcp name=pve01_health target=<PRIVATE_IPV4> value=9101 tcp_9101_rc=0\",\"+OK severity=critical kind=tcp name=pve02_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=tcp name=pve02_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0\",\"+OK severity=critical kind=tcp name=pve03_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=tcp name=pve03_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0\",\"+OK severity=critical kind=tcp name=dns1_tcp53 target=<PRIVATE_IPV4> value=53 tcp_53_rc=0\",\"+OK severity=critical kind=tcp name=dns2_tcp53 target=<PRIVATE_IPV4> value=53 tcp_53_rc=0\",\"+OK severity=critical kind=tcp name=edge_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=tcp name=cluster_admin_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=ping name=vm130_edge_vm_ping target=<PRIVATE_IPV4> value=- ping_rc=0\",\"+OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=ping name=vm150_nextcloud_ping target=<PRIVATE_IPV4> value=- ping_rc=0\",\"+OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1\",\"+INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1\",\"+OK severity=critical kind=ping name=vm160_forum_prod_ping target=<PRIVATE_IPV4> value=- ping_rc=0\",\"+OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=0\",\"+INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1\",\"+OK severity=critical kind=ping name=vm170_core_apps_ping target=<PRIVATE_IPV4> value=- ping_rc=0\",\"+OK severity=critical kind=tcp name=vm170_core_apps_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1\",\"+INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1\",\"+OK severity=critical kind=ping name=vm171_monitoring_ping target=<PRIVATE_IPV4> value=- ping_rc=0\",\"+OK severity=critical kind=tcp name=vm171_monitoring_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1\",\"+INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1\",\" \",\" ## Result\",\"-STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"+STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\" </body>\",\" </html>\"],\"published_lines\":110,\"total_lines\":110,\"truncated\":false},\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"reference_diff\":{\"lines\":[\"--- V19/reference-backend\",\"+++ CURRENT/reference-backend\",\"@@ -31,9 +31,9 @@\",\" [ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\",\" [ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\",\" \",\" if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; COLOR=\\\"#0a7f38\\\"; BADGE=\\\"OK\\\"; else STATUS=WARN; COLOR=\\\"#b26a00\\\"; BADGE=\\\"REVIEW\\\"; fi\",\"-STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\",\"+STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://<PRIVATE_IPV4>:8080/ STATUS_URL=http://<PRIVATE_IPV4>:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\",\" echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\",\" echo \\\"$STATUS_LINE\\\" > \\\"$WEB/status.txt\\\"\",\" \",\" cat > \\\"$WEB/index.html\\\" <<HTML\",\"@@ -55,9 +55,9 @@\",\" </style>\",\" </head>\",\" <body>\",\" <h1>Homelab Cluster Admin</h1>\",\"-<div class=\\\"small\\\">generated UTC: $TS · VM180 · [PRIVATE_IP] · refresh 60s</div>\",\"+<div class=\\\"small\\\">generated UTC: $TS · VM180 · <PRIVATE_IPV4> · refresh 60s</div>\",\" <p><span class=\\\"badge\\\">$BADGE</span></p>\",\" \",\" <div class=\\\"grid\\\">\",\" <div class=\\\"card\\\"><h2>Self</h2><pre>$SELF\"],\"published_lines\":24,\"total_lines\":24,\"truncated\":false},\"remote_identity\":{\"euid\":0,\"hostname\":\"cluster-admin\"},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":22,\"status\":\"OK\",\"transport\":{\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":35786,\"stdout_sha256\":\"cf04063175c8c32ec5f20bb9377e91c2173611df31997715338fcd4f7af95681\"},\"units\":[{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:20:15 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:20:15 UTC\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:15 UTC] ; stop_time=[Wed 2026-08-05 14:20:15 UTC] ; pid=2016430 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"ExecMainExitTimestamp\":\"\",\"ExecMainStartTimestamp\":\"Tue 2026-07-28 11:06:07 UTC\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:20:09 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:20:08 UTC\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:08 UTC] ; stop_time=[Wed 2026-08-05 14:20:09 UTC] ; pid=2016411 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:15:31 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:15:25 UTC\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:31 UTC] ; pid=2015620 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:15:25 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:15:25 UTC\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:25 UTC] ; pid=2015621 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:15:25 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:15:25 UTC\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:25 UTC] ; pid=2015622 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:20:15 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:20:15 UTC\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:15 UTC] ; stop_time=[Wed 2026-08-05 14:20:15 UTC] ; pid=2016431 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"ExecMainExitTimestamp\":\"\",\"ExecMainStartTimestamp\":\"Tue 2026-07-28 11:06:03 UTC\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"failed\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:11:28 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:11:28 UTC\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:11:28 UTC] ; stop_time=[Wed 2026-08-05 14:11:28 UTC] ; pid=2015065 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"LoadState\":\"loaded\",\"Result\":\"exit-code\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\\n\"}],\"v19_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":79419,\"output_sha256\":\"136221f755ac0a721469151448174aef1afce2fb7597afcd8ba4652284099fc9\",\"output_source\":\"runner_json\",\"rollback_restored\":null,\"rollback_started\":false,\"runner_document_rc\":0,\"runner_json_sha256\":\"bc3ce708ad879590b3850b11dd8282ab9cc2faa48274b8dfd3c6a486bd3e1360\",\"runner_rc\":0,\"runner_status\":\"OK\",\"runner_text_sha256\":\"5bab552a2de42048d35e4e73b12267670153bda3b91e14ae8731163b03969f51\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"ba2860a7aab047d7f38ca1141d6f2971e61e990cff504a1f28dce08464c4ecf7\",\"state_path_sha256\":\"f0448a841353b3dc9042f881326d555faaa928689d01b95b4259170feaccbeda\",\"state_status\":\"OK\"},\"v21_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"exact_error\":\"remote python failed: Traceback (most recent call last):\\n  File \\\"<stdin>\\\", line 14, in <module>\\nRuntimeError: sha drift /var/www/homelab-cluster-admin/index.html\\n\",\"exact_error_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":843,\"output_sha256\":\"dcd6c5631069f8d522b62ac544e52af1fd03f187bb862ecfa240fdf6b628b199\",\"output_source\":\"runner_json\",\"rollback_restored\":false,\"rollback_started\":false,\"runner_document_rc\":1,\"runner_json_sha256\":\"3ad8f45bdaff6a3b68a44a0c1bb22241dcd39e115728837ae5594b292321898e\",\"runner_rc\":1,\"runner_status\":\"FAIL\",\"runner_text_sha256\":\"78d8d650581eafcde8a7ab1498fc2269bb3c4b3b46ca0e388401f3db88164771\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"555ae2a4e44cf6d6dac5216dfd5a8f58d4fdcf667392c3c03d3f79f099655fca\",\"state_path_sha256\":\"1e2f176c02f8bd3d63a101110406e855e05023596ee5f05a49ee4356d8c84f8a\",\"state_status\":\"FAIL\"}}\n"
+}
diff --git a/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z.txt b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z.txt
new file mode 100644
index 0000000..65b20f4
--- /dev/null
+++ b/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z.txt
@@ -0,0 +1,34 @@
+CHAT_OUTPUT_BEGIN
+COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z
+STATUS=OK
+RC=0
+HOST=pve01
+MODE=verify
+COMPONENT=homelab-cluster-admin-update-visibility
+REFERENCE_REGISTER_CHECK=OK
+REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
+ERROR_REGISTER_CHECK=OK
+ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
+COMMAND_SHA256=b4bce2469c19f1a3186f956193115529e14e4df31c9798c2de3a08fa8dd1c9ef
+DUPLICATE_FAILED_COMMAND_BLOCKED=false
+EXECUTION_STARTED=true
+CHANGE_DECLARED=false
+RESULT_CONTRACT_VALID=true
+RESULT_CONTRACT_STATUS=NOT_APPLICABLE
+RESULT_CONTRACT_ERROR=NONE
+COMMAND_RC=0
+CHANGES_MADE=false
+ROLLBACK_STARTED=false
+ROLLBACK_RESTORED=null
+MUTATION_OUTCOME=NO_MUTATION
+SANITIZED=yes
+SECRETS_INCLUDED=no
+PRIVATE_ADDRESSES_INCLUDED=no
+RAW_EVIDENCE_SHA256=81f3a7b84d9eec86671f34f89bc59d99a95dd08ec9d6911e6ea399964e0e9718
+SANITIZED_OUTPUT_SHA256=691a9fb3ca3ff21163b447d98d7f8055f3e7ad70e86dafd2b73cdd5e9d118969
+OUTPUT_BEGIN
+HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z","command_rc":0,"control_plane_error":null,"index_changed_diff_lines":110,"index_current_sha256":"686e55e3934e22180a3889a0ee9ba8cdcbb92b5d9d1bc5418080f701a5796172","index_drift":true,"index_expected_sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","index_path":"/var/www/homelab-cluster-admin/index.html","managed_marker_present":false,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_corrected_transaction":true,"reference_current_sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","reference_drift":false,"reference_expected_sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","remote_probe_root":true,"report_bytes":33176,"report_sha256":"04df393f3f5a7703399153a129a8458a3c9980f45c4cb00b1f9bd8b62182c15c","rollback_restored":false,"rollback_started":false,"status":"OK","v19_exact_result_identity_confirmed":true,"v21_exact_result_identity_confirmed":true,"v21_exact_sha_drift_confirmed":true,"version":1}
+{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z","current_files":{"index":{"backend_marker_begin":false,"backend_marker_end":false,"bytes":10353,"ctime_ns":1785939615262660290,"expected_sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","gid":0,"index_marker_begin":false,"index_marker_end":false,"inode":135903,"matches_v19":false,"mode":420,"mtime_ns":1785939615262660290,"path":"/var/www/homelab-cluster-admin/index.html","sha256":"686e55e3934e22180a3889a0ee9ba8cdcbb92b5d9d1bc5418080f701a5796172","uid":0},"reference":{"backend_marker_begin":false,"backend_marker_end":false,"bytes":3708,"ctime_ns":1783706500680296060,"expected_sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","gid":0,"index_marker_begin":false,"index_marker_end":false,"inode":135901,"matches_v19":true,"mode":493,"mtime_ns":1783706500680296060,"path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","uid":0}},"drift_analysis":{"index_changed_diff_lines":110,"index_drift":true,"managed_marker_present":false,"next_required_action":"build corrected atomic transaction using current index SHA","ready_for_corrected_transaction":true,"reference_changed_diff_lines":24,"reference_drift":false},"index_diff":{"lines":["--- V19/index.html","+++ CURRENT/index.html","@@ -16,65 +16,65 @@"," </style>"," </head>"," <body>"," <h1>Homelab Cluster Admin</h1>","-<div class=\"small\">generated UTC: 2026-08-05T13:20:48Z · VM180 · [PRIVATE_IP] · refresh 60s</div>","+<div class=\"small\">generated UTC: 2026-08-05T14:20:15Z · VM180 · <PRIVATE_IPV4> · refresh 60s</div>"," <p><span class=\"badge\">REVIEW</span></p>"," "," <div class=\"grid\">","-<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","+<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"," FAILED_UNITS=1"," QGA=active</pre></div>","-<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","-<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","+<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","+<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"," <div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"," </div>"," ","-<div class=<REDACTED>","+<div class=<REDACTED>"," <div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>","-<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>","+<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=<PRIVATE_IPV4> MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"," <div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer","-generated_utc=2026-08-05T13:19:08Z","+generated_utc=2026-08-05T14:15:25Z"," policy=critical_failures_are_bad_optional_tcp_is_inventory_info"," "," ## Restricted probes","-pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","-pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2","-pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0","-edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active","+pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","+pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T14:15:27Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2","+pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:15:29Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0","+edge ssh_rc=0 STATUS=OK TS=2026-08-05T14:15:31Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"," "," ## Service map checks","-OK severity=<REDACTED>","-OK severity=critical kind=tcp name=pve01_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=tcp name=pve01_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0","-OK severity=critical kind=tcp name=pve01_health target=[PRIVATE_IP] value=9101 tcp_9101_rc=0","-OK severity=critical kind=tcp name=pve02_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=tcp name=pve02_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0","-OK severity=critical kind=tcp name=pve03_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=tcp name=pve03_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0","-OK severity=critical kind=tcp name=dns1_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0","-OK severity=critical kind=tcp name=dns2_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0","-OK severity=critical kind=tcp name=edge_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=ping name=vm130_edge_vm_ping target=[PRIVATE_IP] value=- ping_rc=0","-OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=ping name=vm150_nextcloud_ping target=[PRIVATE_IP] value=- ping_rc=0","-OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1","-INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1","-OK severity=critical kind=ping name=vm160_forum_prod_ping target=[PRIVATE_IP] value=- ping_rc=0","-OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=0","-INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1","-OK severity=critical kind=ping name=vm170_core_apps_ping target=[PRIVATE_IP] value=- ping_rc=0","-OK severity=critical kind=tcp name=vm170_core_apps_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1","-INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1","-OK severity=critical kind=ping name=vm171_monitoring_ping target=[PRIVATE_IP] value=- ping_rc=0","-OK severity=critical kind=tcp name=vm171_monitoring_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1","-INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1","+OK severity=<REDACTED>","+OK severity=critical kind=tcp name=pve01_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=tcp name=pve01_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0","+OK severity=critical kind=tcp name=pve01_health target=<PRIVATE_IPV4> value=9101 tcp_9101_rc=0","+OK severity=critical kind=tcp name=pve02_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=tcp name=pve02_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0","+OK severity=critical kind=tcp name=pve03_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=tcp name=pve03_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0","+OK severity=critical kind=tcp name=dns1_tcp53 target=<PRIVATE_IPV4> value=53 tcp_53_rc=0","+OK severity=critical kind=tcp name=dns2_tcp53 target=<PRIVATE_IPV4> value=53 tcp_53_rc=0","+OK severity=critical kind=tcp name=edge_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=tcp name=cluster_admin_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=ping name=vm130_edge_vm_ping target=<PRIVATE_IPV4> value=- ping_rc=0","+OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=ping name=vm150_nextcloud_ping target=<PRIVATE_IPV4> value=- ping_rc=0","+OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1","+INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1","+OK severity=critical kind=ping name=vm160_forum_prod_ping target=<PRIVATE_IPV4> value=- ping_rc=0","+OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=0","+INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1","+OK severity=critical kind=ping name=vm170_core_apps_ping target=<PRIVATE_IPV4> value=- ping_rc=0","+OK severity=critical kind=tcp name=vm170_core_apps_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1","+INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1","+OK severity=critical kind=ping name=vm171_monitoring_ping target=<PRIVATE_IPV4> value=- ping_rc=0","+OK severity=critical kind=tcp name=vm171_monitoring_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1","+INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1"," "," ## Result","-STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","+STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"," </body>"," </html>"],"published_lines":110,"total_lines":110,"truncated":false},"mutation_outcome":"NO_MUTATION","read_only":true,"reference_diff":{"lines":["--- V19/reference-backend","+++ CURRENT/reference-backend","@@ -31,9 +31,9 @@"," [ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"," [ \"$QGA\" = active ] || BAD=$((BAD+1))"," "," if [ \"$BAD\" = 0 ]; then STATUS=OK; COLOR=\"#0a7f38\"; BADGE=\"OK\"; else STATUS=WARN; COLOR=\"#b26a00\"; BADGE=\"REVIEW\"; fi","-STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"","+STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://<PRIVATE_IPV4>:8080/ STATUS_URL=http://<PRIVATE_IPV4>:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""," echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"," echo \"$STATUS_LINE\" > \"$WEB/status.txt\""," "," cat > \"$WEB/index.html\" <<HTML","@@ -55,9 +55,9 @@"," </style>"," </head>"," <body>"," <h1>Homelab Cluster Admin</h1>","-<div class=\"small\">generated UTC: $TS · VM180 · [PRIVATE_IP] · refresh 60s</div>","+<div class=\"small\">generated UTC: $TS · VM180 · <PRIVATE_IPV4> · refresh 60s</div>"," <p><span class=\"badge\">$BADGE</span></p>"," "," <div class=\"grid\">"," <div class=\"card\"><h2>Self</h2><pre>$SELF"],"published_lines":24,"total_lines":24,"truncated":false},"remote_identity":{"euid":0,"hostname":"cluster-admin"},"rollback_restored":false,"rollback_started":false,"schema":22,"status":"OK","transport":{"rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":35786,"stdout_sha256":"cf04063175c8c32ec5f20bb9377e91c2173611df31997715338fcd4f7af95681"},"units":[{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:20:15 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:20:15 UTC","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:15 UTC] ; stop_time=[Wed 2026-08-05 14:20:15 UTC] ; pid=2016430 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","ExecMainExitTimestamp":"","ExecMainStartTimestamp":"Tue 2026-07-28 11:06:07 UTC","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:20:09 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:20:08 UTC","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:08 UTC] ; stop_time=[Wed 2026-08-05 14:20:09 UTC] ; pid=2016411 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:15:31 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:15:25 UTC","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:31 UTC] ; pid=2015620 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:15:25 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:15:25 UTC","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:25 UTC] ; pid=2015621 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:15:25 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:15:25 UTC","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:25 UTC] ; pid=2015622 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:20:15 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:20:15 UTC","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:15 UTC] ; stop_time=[Wed 2026-08-05 14:20:15 UTC] ; pid=2016431 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","ExecMainExitTimestamp":"","ExecMainStartTimestamp":"Tue 2026-07-28 11:06:03 UTC","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"failed","ExecMainExitTimestamp":"Wed 2026-08-05 14:11:28 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:11:28 UTC","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:11:28 UTC] ; stop_time=[Wed 2026-08-05 14:11:28 UTC] ; pid=2015065 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","LoadState":"loaded","Result":"exit-code","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target\n"}],"v19_exact_result":{"changes_made":false,"control_plane_error":null,"mutation_outcome":"NO_MUTATION","output_bytes":79419,"output_sha256":"136221f755ac0a721469151448174aef1afce2fb7597afcd8ba4652284099fc9","output_source":"runner_json","rollback_restored":null,"rollback_started":false,"runner_document_rc":0,"runner_json_sha256":"bc3ce708ad879590b3850b11dd8282ab9cc2faa48274b8dfd3c6a486bd3e1360","runner_rc":0,"runner_status":"OK","runner_text_sha256":"5bab552a2de42048d35e4e73b12267670153bda3b91e14ae8731163b03969f51","runner_wrapper_matches_document":true,"state_document_sha256":"ba2860a7aab047d7f38ca1141d6f2971e61e990cff504a1f28dce08464c4ecf7","state_path_sha256":"f0448a841353b3dc9042f881326d555faaa928689d01b95b4259170feaccbeda","state_status":"OK"},"v21_exact_result":{"changes_made":false,"control_plane_error":null,"exact_error":"remote python failed: Traceback (most recent call last):\n  File \"<stdin>\", line 14, in <module>\nRuntimeError: sha drift /var/www/homelab-cluster-admin/index.html\n","exact_error_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":843,"output_sha256":"dcd6c5631069f8d522b62ac544e52af1fd03f187bb862ecfa240fdf6b628b199","output_source":"runner_json","rollback_restored":false,"rollback_started":false,"runner_document_rc":1,"runner_json_sha256":"3ad8f45bdaff6a3b68a44a0c1bb22241dcd39e115728837ae5594b292321898e","runner_rc":1,"runner_status":"FAIL","runner_text_sha256":"78d8d650581eafcde8a7ab1498fc2269bb3c4b3b46ca0e388401f3db88164771","runner_wrapper_matches_document":true,"state_document_sha256":"555ae2a4e44cf6d6dac5216dfd5a8f58d4fdcf667392c3c03d3f79f099655fca","state_path_sha256":"1e2f176c02f8bd3d63a101110406e855e05023596ee5f05a49ee4356d8c84f8a","state_status":"FAIL"}}
+
+OUTPUT_END
+CHAT_OUTPUT_END
diff --git a/runtime/latest.json b/runtime/latest.json
index 7fe18b1..27c8a75 100644
--- a/runtime/latest.json
+++ b/runtime/latest.json
@@ -1,38 +1,38 @@
 {
   "schema_version": 1,
   "channel": "homelab-runtime",
-  "command_id": "HOMELAB-SKLADCHIK-SECRET-HANDOFF-INSTALL-V2-20260805T141500Z",
-  "status": "FAIL",
-  "rc": 76,
+  "command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z",
+  "status": "OK",
+  "rc": 0,
   "host": "pve01",
-  "mode": "change",
-  "component": "skladchik-secret-handoff-install",
-  "started_at_utc": "2026-08-05T14:18:43Z",
-  "finished_at_utc": "2026-08-05T14:18:43Z",
+  "mode": "verify",
+  "component": "homelab-cluster-admin-update-visibility",
+  "started_at_utc": "2026-08-05T14:20:20Z",
+  "finished_at_utc": "2026-08-05T14:20:20Z",
   "reference_register_checked": true,
   "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
   "error_register_checked": true,
   "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
-  "command_sha256": "b6a2aa13e3cfd9b11d1bf00bbc382851f2c5e8b03e383737cb49a5223f21028b",
+  "command_sha256": "b4bce2469c19f1a3186f956193115529e14e4df31c9798c2de3a08fa8dd1c9ef",
   "duplicate_failed_command_blocked": false,
   "block_reason": null,
   "execution_started": true,
-  "change_declared": true,
-  "result_contract_valid": false,
+  "change_declared": false,
+  "result_contract_valid": true,
   "result_contract_status": null,
-  "result_contract_error": "MISSING_CONTRACT",
-  "command_rc": 2,
-  "changes_made": null,
-  "rollback_started": null,
+  "result_contract_error": null,
+  "command_rc": 0,
+  "changes_made": false,
+  "rollback_started": false,
   "rollback_restored": null,
-  "mutation_outcome": "UNKNOWN",
+  "mutation_outcome": "NO_MUTATION",
   "sanitized": true,
   "secrets_included": false,
   "private_addresses_included": false,
   "raw_evidence_retained_locally": true,
-  "raw_evidence_sha256": "d9cd93ca143adf160c758f9e73f9916a978424d7224b5822894c4e8a345e5fcf",
-  "sanitized_output_sha256": "d9cd93ca143adf160c758f9e73f9916a978424d7224b5822894c4e8a345e5fcf",
+  "raw_evidence_sha256": "81f3a7b84d9eec86671f34f89bc59d99a95dd08ec9d6911e6ea399964e0e9718",
+  "sanitized_output_sha256": "691a9fb3ca3ff21163b447d98d7f8055f3e7ad70e86dafd2b73cdd5e9d118969",
   "output_truncated_in_json": false,
   "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
-  "output": "/usr/bin/python3: can't open file '//skladchik_secret_handoff_install_v2.py': [Errno 2] No such file or directory\n"
+  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z\",\"command_rc\":0,\"control_plane_error\":null,\"index_changed_diff_lines\":110,\"index_current_sha256\":\"686e55e3934e22180a3889a0ee9ba8cdcbb92b5d9d1bc5418080f701a5796172\",\"index_drift\":true,\"index_expected_sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"index_path\":\"/var/www/homelab-cluster-admin/index.html\",\"managed_marker_present\":false,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_corrected_transaction\":true,\"reference_current_sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"reference_drift\":false,\"reference_expected_sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"remote_probe_root\":true,\"report_bytes\":33176,\"report_sha256\":\"04df393f3f5a7703399153a129a8458a3c9980f45c4cb00b1f9bd8b62182c15c\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"v19_exact_result_identity_confirmed\":true,\"v21_exact_result_identity_confirmed\":true,\"v21_exact_sha_drift_confirmed\":true,\"version\":1}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z\",\"current_files\":{\"index\":{\"backend_marker_begin\":false,\"backend_marker_end\":false,\"bytes\":10353,\"ctime_ns\":1785939615262660290,\"expected_sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"gid\":0,\"index_marker_begin\":false,\"index_marker_end\":false,\"inode\":135903,\"matches_v19\":false,\"mode\":420,\"mtime_ns\":1785939615262660290,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"686e55e3934e22180a3889a0ee9ba8cdcbb92b5d9d1bc5418080f701a5796172\",\"uid\":0},\"reference\":{\"backend_marker_begin\":false,\"backend_marker_end\":false,\"bytes\":3708,\"ctime_ns\":1783706500680296060,\"expected_sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"gid\":0,\"index_marker_begin\":false,\"index_marker_end\":false,\"inode\":135901,\"matches_v19\":true,\"mode\":493,\"mtime_ns\":1783706500680296060,\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"uid\":0}},\"drift_analysis\":{\"index_changed_diff_lines\":110,\"index_drift\":true,\"managed_marker_present\":false,\"next_required_action\":\"build corrected atomic transaction using current index SHA\",\"ready_for_corrected_transaction\":true,\"reference_changed_diff_lines\":24,\"reference_drift\":false},\"index_diff\":{\"lines\":[\"--- V19/index.html\",\"+++ CURRENT/index.html\",\"@@ -16,65 +16,65 @@\",\" </style>\",\" </head>\",\" <body>\",\" <h1>Homelab Cluster Admin</h1>\",\"-<div class=\\\"small\\\">generated UTC: 2026-08-05T13:20:48Z · VM180 · [PRIVATE_IP] · refresh 60s</div>\",\"+<div class=\\\"small\\\">generated UTC: 2026-08-05T14:20:15Z · VM180 · <PRIVATE_IPV4> · refresh 60s</div>\",\" <p><span class=\\\"badge\\\">REVIEW</span></p>\",\" \",\" <div class=\\\"grid\\\">\",\"-<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"+<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\" FAILED_UNITS=1\",\" QGA=active</pre></div>\",\"-<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"-<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"+<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"+<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\" <div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\",\" </div>\",\" \",\"-<div class=<REDACTED>\",\"+<div class=<REDACTED>\",\" <div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\",\"-<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\"+<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=<PRIVATE_IPV4> MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\" <div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\",\"-generated_utc=2026-08-05T13:19:08Z\",\"+generated_utc=2026-08-05T14:15:25Z\",\" policy=critical_failures_are_bad_optional_tcp_is_inventory_info\",\" \",\" ## Restricted probes\",\"-pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"-pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\",\"-pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\",\"-edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\",\"+pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"+pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T14:15:27Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\",\"+pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:15:29Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\",\"+edge ssh_rc=0 STATUS=OK TS=2026-08-05T14:15:31Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\",\" \",\" ## Service map checks\",\"-OK severity=<REDACTED>\",\"-OK severity=critical kind=tcp name=pve01_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=tcp name=pve01_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\",\"-OK severity=critical kind=tcp name=pve01_health target=[PRIVATE_IP] value=9101 tcp_9101_rc=0\",\"-OK severity=critical kind=tcp name=pve02_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=tcp name=pve02_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\",\"-OK severity=critical kind=tcp name=pve03_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=tcp name=pve03_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\",\"-OK severity=critical kind=tcp name=dns1_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0\",\"-OK severity=critical kind=tcp name=dns2_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0\",\"-OK severity=critical kind=tcp name=edge_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=ping name=vm130_edge_vm_ping target=[PRIVATE_IP] value=- ping_rc=0\",\"-OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=critical kind=ping name=vm150_nextcloud_ping target=[PRIVATE_IP] value=- ping_rc=0\",\"-OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\",\"-INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\",\"-OK severity=critical kind=ping name=vm160_forum_prod_ping target=[PRIVATE_IP] value=- ping_rc=0\",\"-OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=0\",\"-INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\",\"-OK severity=critical kind=ping name=vm170_core_apps_ping target=[PRIVATE_IP] value=- ping_rc=0\",\"-OK severity=critical kind=tcp name=vm170_core_apps_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\",\"-INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\",\"-OK severity=critical kind=ping name=vm171_monitoring_ping target=[PRIVATE_IP] value=- ping_rc=0\",\"-OK severity=critical kind=tcp name=vm171_monitoring_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"-INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\",\"-INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\",\"+OK severity=<REDACTED>\",\"+OK severity=critical kind=tcp name=pve01_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=tcp name=pve01_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0\",\"+OK severity=critical kind=tcp name=pve01_health target=<PRIVATE_IPV4> value=9101 tcp_9101_rc=0\",\"+OK severity=critical kind=tcp name=pve02_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=tcp name=pve02_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0\",\"+OK severity=critical kind=tcp name=pve03_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=tcp name=pve03_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0\",\"+OK severity=critical kind=tcp name=dns1_tcp53 target=<PRIVATE_IPV4> value=53 tcp_53_rc=0\",\"+OK severity=critical kind=tcp name=dns2_tcp53 target=<PRIVATE_IPV4> value=53 tcp_53_rc=0\",\"+OK severity=critical kind=tcp name=edge_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=tcp name=cluster_admin_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=ping name=vm130_edge_vm_ping target=<PRIVATE_IPV4> value=- ping_rc=0\",\"+OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=critical kind=ping name=vm150_nextcloud_ping target=<PRIVATE_IPV4> value=- ping_rc=0\",\"+OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1\",\"+INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1\",\"+OK severity=critical kind=ping name=vm160_forum_prod_ping target=<PRIVATE_IPV4> value=- ping_rc=0\",\"+OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=0\",\"+INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1\",\"+OK severity=critical kind=ping name=vm170_core_apps_ping target=<PRIVATE_IPV4> value=- ping_rc=0\",\"+OK severity=critical kind=tcp name=vm170_core_apps_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1\",\"+INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1\",\"+OK severity=critical kind=ping name=vm171_monitoring_ping target=<PRIVATE_IPV4> value=- ping_rc=0\",\"+OK severity=critical kind=tcp name=vm171_monitoring_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0\",\"+INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1\",\"+INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1\",\" \",\" ## Result\",\"-STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"+STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\" </body>\",\" </html>\"],\"published_lines\":110,\"total_lines\":110,\"truncated\":false},\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"reference_diff\":{\"lines\":[\"--- V19/reference-backend\",\"+++ CURRENT/reference-backend\",\"@@ -31,9 +31,9 @@\",\" [ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\",\" [ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\",\" \",\" if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; COLOR=\\\"#0a7f38\\\"; BADGE=\\\"OK\\\"; else STATUS=WARN; COLOR=\\\"#b26a00\\\"; BADGE=\\\"REVIEW\\\"; fi\",\"-STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\",\"+STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://<PRIVATE_IPV4>:8080/ STATUS_URL=http://<PRIVATE_IPV4>:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\",\" echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\",\" echo \\\"$STATUS_LINE\\\" > \\\"$WEB/status.txt\\\"\",\" \",\" cat > \\\"$WEB/index.html\\\" <<HTML\",\"@@ -55,9 +55,9 @@\",\" </style>\",\" </head>\",\" <body>\",\" <h1>Homelab Cluster Admin</h1>\",\"-<div class=\\\"small\\\">generated UTC: $TS · VM180 · [PRIVATE_IP] · refresh 60s</div>\",\"+<div class=\\\"small\\\">generated UTC: $TS · VM180 · <PRIVATE_IPV4> · refresh 60s</div>\",\" <p><span class=\\\"badge\\\">$BADGE</span></p>\",\" \",\" <div class=\\\"grid\\\">\",\" <div class=\\\"card\\\"><h2>Self</h2><pre>$SELF\"],\"published_lines\":24,\"total_lines\":24,\"truncated\":false},\"remote_identity\":{\"euid\":0,\"hostname\":\"cluster-admin\"},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":22,\"status\":\"OK\",\"transport\":{\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":35786,\"stdout_sha256\":\"cf04063175c8c32ec5f20bb9377e91c2173611df31997715338fcd4f7af95681\"},\"units\":[{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:20:15 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:20:15 UTC\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:15 UTC] ; stop_time=[Wed 2026-08-05 14:20:15 UTC] ; pid=2016430 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"ExecMainExitTimestamp\":\"\",\"ExecMainStartTimestamp\":\"Tue 2026-07-28 11:06:07 UTC\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:20:09 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:20:08 UTC\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:08 UTC] ; stop_time=[Wed 2026-08-05 14:20:09 UTC] ; pid=2016411 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:15:31 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:15:25 UTC\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:31 UTC] ; pid=2015620 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:15:25 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:15:25 UTC\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:25 UTC] ; pid=2015621 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:15:25 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:15:25 UTC\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:25 UTC] ; pid=2015622 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:20:15 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:20:15 UTC\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:15 UTC] ; stop_time=[Wed 2026-08-05 14:20:15 UTC] ; pid=2016431 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"ExecMainExitTimestamp\":\"\",\"ExecMainStartTimestamp\":\"Tue 2026-07-28 11:06:03 UTC\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"failed\",\"ExecMainExitTimestamp\":\"Wed 2026-08-05 14:11:28 UTC\",\"ExecMainStartTimestamp\":\"Wed 2026-08-05 14:11:28 UTC\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:11:28 UTC] ; stop_time=[Wed 2026-08-05 14:11:28 UTC] ; pid=2015065 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"LoadState\":\"loaded\",\"Result\":\"exit-code\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\\n\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\\n\"}],\"v19_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":79419,\"output_sha256\":\"136221f755ac0a721469151448174aef1afce2fb7597afcd8ba4652284099fc9\",\"output_source\":\"runner_json\",\"rollback_restored\":null,\"rollback_started\":false,\"runner_document_rc\":0,\"runner_json_sha256\":\"bc3ce708ad879590b3850b11dd8282ab9cc2faa48274b8dfd3c6a486bd3e1360\",\"runner_rc\":0,\"runner_status\":\"OK\",\"runner_text_sha256\":\"5bab552a2de42048d35e4e73b12267670153bda3b91e14ae8731163b03969f51\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"ba2860a7aab047d7f38ca1141d6f2971e61e990cff504a1f28dce08464c4ecf7\",\"state_path_sha256\":\"f0448a841353b3dc9042f881326d555faaa928689d01b95b4259170feaccbeda\",\"state_status\":\"OK\"},\"v21_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"exact_error\":\"remote python failed: Traceback (most recent call last):\\n  File \\\"<stdin>\\\", line 14, in <module>\\nRuntimeError: sha drift /var/www/homelab-cluster-admin/index.html\\n\",\"exact_error_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":843,\"output_sha256\":\"dcd6c5631069f8d522b62ac544e52af1fd03f187bb862ecfa240fdf6b628b199\",\"output_source\":\"runner_json\",\"rollback_restored\":false,\"rollback_started\":false,\"runner_document_rc\":1,\"runner_json_sha256\":\"3ad8f45bdaff6a3b68a44a0c1bb22241dcd39e115728837ae5594b292321898e\",\"runner_rc\":1,\"runner_status\":\"FAIL\",\"runner_text_sha256\":\"78d8d650581eafcde8a7ab1498fc2269bb3c4b3b46ca0e388401f3db88164771\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"555ae2a4e44cf6d6dac5216dfd5a8f58d4fdcf667392c3c03d3f79f099655fca\",\"state_path_sha256\":\"1e2f176c02f8bd3d63a101110406e855e05023596ee5f05a49ee4356d8c84f8a\",\"state_status\":\"FAIL\"}}\n"
 }
diff --git a/runtime/latest.txt b/runtime/latest.txt
index 82a5545..65b20f4 100644
--- a/runtime/latest.txt
+++ b/runtime/latest.txt
@@ -1,33 +1,34 @@
 CHAT_OUTPUT_BEGIN
-COMMAND_ID=HOMELAB-SKLADCHIK-SECRET-HANDOFF-INSTALL-V2-20260805T141500Z
-STATUS=FAIL
-RC=76
+COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z
+STATUS=OK
+RC=0
 HOST=pve01
-MODE=change
-COMPONENT=skladchik-secret-handoff-install
+MODE=verify
+COMPONENT=homelab-cluster-admin-update-visibility
 REFERENCE_REGISTER_CHECK=OK
 REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
 ERROR_REGISTER_CHECK=OK
 ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
-COMMAND_SHA256=b6a2aa13e3cfd9b11d1bf00bbc382851f2c5e8b03e383737cb49a5223f21028b
+COMMAND_SHA256=b4bce2469c19f1a3186f956193115529e14e4df31c9798c2de3a08fa8dd1c9ef
 DUPLICATE_FAILED_COMMAND_BLOCKED=false
 EXECUTION_STARTED=true
-CHANGE_DECLARED=true
-RESULT_CONTRACT_VALID=false
-RESULT_CONTRACT_STATUS=UNKNOWN
-RESULT_CONTRACT_ERROR=MISSING_CONTRACT
-COMMAND_RC=2
-CHANGES_MADE=unknown
-ROLLBACK_STARTED=unknown
+CHANGE_DECLARED=false
+RESULT_CONTRACT_VALID=true
+RESULT_CONTRACT_STATUS=NOT_APPLICABLE
+RESULT_CONTRACT_ERROR=NONE
+COMMAND_RC=0
+CHANGES_MADE=false
+ROLLBACK_STARTED=false
 ROLLBACK_RESTORED=null
-MUTATION_OUTCOME=UNKNOWN
+MUTATION_OUTCOME=NO_MUTATION
 SANITIZED=yes
 SECRETS_INCLUDED=no
 PRIVATE_ADDRESSES_INCLUDED=no
-RAW_EVIDENCE_SHA256=d9cd93ca143adf160c758f9e73f9916a978424d7224b5822894c4e8a345e5fcf
-SANITIZED_OUTPUT_SHA256=d9cd93ca143adf160c758f9e73f9916a978424d7224b5822894c4e8a345e5fcf
+RAW_EVIDENCE_SHA256=81f3a7b84d9eec86671f34f89bc59d99a95dd08ec9d6911e6ea399964e0e9718
+SANITIZED_OUTPUT_SHA256=691a9fb3ca3ff21163b447d98d7f8055f3e7ad70e86dafd2b73cdd5e9d118969
 OUTPUT_BEGIN
-/usr/bin/python3: can't open file '//skladchik_secret_handoff_install_v2.py': [Errno 2] No such file or directory
+HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z","command_rc":0,"control_plane_error":null,"index_changed_diff_lines":110,"index_current_sha256":"686e55e3934e22180a3889a0ee9ba8cdcbb92b5d9d1bc5418080f701a5796172","index_drift":true,"index_expected_sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","index_path":"/var/www/homelab-cluster-admin/index.html","managed_marker_present":false,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_corrected_transaction":true,"reference_current_sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","reference_drift":false,"reference_expected_sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","remote_probe_root":true,"report_bytes":33176,"report_sha256":"04df393f3f5a7703399153a129a8458a3c9980f45c4cb00b1f9bd8b62182c15c","rollback_restored":false,"rollback_started":false,"status":"OK","v19_exact_result_identity_confirmed":true,"v21_exact_result_identity_confirmed":true,"v21_exact_sha_drift_confirmed":true,"version":1}
+{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V22-INDEX-DRIFT-READ-ONLY-20260805T142000Z","current_files":{"index":{"backend_marker_begin":false,"backend_marker_end":false,"bytes":10353,"ctime_ns":1785939615262660290,"expected_sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","gid":0,"index_marker_begin":false,"index_marker_end":false,"inode":135903,"matches_v19":false,"mode":420,"mtime_ns":1785939615262660290,"path":"/var/www/homelab-cluster-admin/index.html","sha256":"686e55e3934e22180a3889a0ee9ba8cdcbb92b5d9d1bc5418080f701a5796172","uid":0},"reference":{"backend_marker_begin":false,"backend_marker_end":false,"bytes":3708,"ctime_ns":1783706500680296060,"expected_sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","gid":0,"index_marker_begin":false,"index_marker_end":false,"inode":135901,"matches_v19":true,"mode":493,"mtime_ns":1783706500680296060,"path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","uid":0}},"drift_analysis":{"index_changed_diff_lines":110,"index_drift":true,"managed_marker_present":false,"next_required_action":"build corrected atomic transaction using current index SHA","ready_for_corrected_transaction":true,"reference_changed_diff_lines":24,"reference_drift":false},"index_diff":{"lines":["--- V19/index.html","+++ CURRENT/index.html","@@ -16,65 +16,65 @@"," </style>"," </head>"," <body>"," <h1>Homelab Cluster Admin</h1>","-<div class=\"small\">generated UTC: 2026-08-05T13:20:48Z · VM180 · [PRIVATE_IP] · refresh 60s</div>","+<div class=\"small\">generated UTC: 2026-08-05T14:20:15Z · VM180 · <PRIVATE_IPV4> · refresh 60s</div>"," <p><span class=\"badge\">REVIEW</span></p>"," "," <div class=\"grid\">","-<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","+<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"," FAILED_UNITS=1"," QGA=active</pre></div>","-<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","-<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","+<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","+<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"," <div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"," </div>"," ","-<div class=<REDACTED>","+<div class=<REDACTED>"," <div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>","-<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>","+<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=<PRIVATE_IPV4> MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"," <div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer","-generated_utc=2026-08-05T13:19:08Z","+generated_utc=2026-08-05T14:15:25Z"," policy=critical_failures_are_bad_optional_tcp_is_inventory_info"," "," ## Restricted probes","-pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","-pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2","-pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0","-edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active","+pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","+pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T14:15:27Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2","+pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:15:29Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0","+edge ssh_rc=0 STATUS=OK TS=2026-08-05T14:15:31Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"," "," ## Service map checks","-OK severity=<REDACTED>","-OK severity=critical kind=tcp name=pve01_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=tcp name=pve01_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0","-OK severity=critical kind=tcp name=pve01_health target=[PRIVATE_IP] value=9101 tcp_9101_rc=0","-OK severity=critical kind=tcp name=pve02_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=tcp name=pve02_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0","-OK severity=critical kind=tcp name=pve03_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=tcp name=pve03_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0","-OK severity=critical kind=tcp name=dns1_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0","-OK severity=critical kind=tcp name=dns2_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0","-OK severity=critical kind=tcp name=edge_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=ping name=vm130_edge_vm_ping target=[PRIVATE_IP] value=- ping_rc=0","-OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=critical kind=ping name=vm150_nextcloud_ping target=[PRIVATE_IP] value=- ping_rc=0","-OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1","-INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1","-OK severity=critical kind=ping name=vm160_forum_prod_ping target=[PRIVATE_IP] value=- ping_rc=0","-OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=0","-INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1","-OK severity=critical kind=ping name=vm170_core_apps_ping target=[PRIVATE_IP] value=- ping_rc=0","-OK severity=critical kind=tcp name=vm170_core_apps_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1","-INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1","-OK severity=critical kind=ping name=vm171_monitoring_ping target=[PRIVATE_IP] value=- ping_rc=0","-OK severity=critical kind=tcp name=vm171_monitoring_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","-INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1","-INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1","+OK severity=<REDACTED>","+OK severity=critical kind=tcp name=pve01_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=tcp name=pve01_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0","+OK severity=critical kind=tcp name=pve01_health target=<PRIVATE_IPV4> value=9101 tcp_9101_rc=0","+OK severity=critical kind=tcp name=pve02_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=tcp name=pve02_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0","+OK severity=critical kind=tcp name=pve03_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=tcp name=pve03_proxmox target=<PRIVATE_IPV4> value=8006 tcp_8006_rc=0","+OK severity=critical kind=tcp name=dns1_tcp53 target=<PRIVATE_IPV4> value=53 tcp_53_rc=0","+OK severity=critical kind=tcp name=dns2_tcp53 target=<PRIVATE_IPV4> value=53 tcp_53_rc=0","+OK severity=critical kind=tcp name=edge_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=tcp name=cluster_admin_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=ping name=vm130_edge_vm_ping target=<PRIVATE_IPV4> value=- ping_rc=0","+OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=critical kind=ping name=vm150_nextcloud_ping target=<PRIVATE_IPV4> value=- ping_rc=0","+OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1","+INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1","+OK severity=critical kind=ping name=vm160_forum_prod_ping target=<PRIVATE_IPV4> value=- ping_rc=0","+OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=0","+INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1","+OK severity=critical kind=ping name=vm170_core_apps_ping target=<PRIVATE_IPV4> value=- ping_rc=0","+OK severity=critical kind=tcp name=vm170_core_apps_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1","+INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1","+OK severity=critical kind=ping name=vm171_monitoring_ping target=<PRIVATE_IPV4> value=- ping_rc=0","+OK severity=critical kind=tcp name=vm171_monitoring_ssh target=<PRIVATE_IPV4> value=22 tcp_22_rc=0","+INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=<PRIVATE_IPV4> value=80 tcp_80_rc=1","+INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=<PRIVATE_IPV4> value=443 tcp_443_rc=1"," "," ## Result","-STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","+STATUS=WARN TS=2026-08-05T14:15:25Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"," </body>"," </html>"],"published_lines":110,"total_lines":110,"truncated":false},"mutation_outcome":"NO_MUTATION","read_only":true,"reference_diff":{"lines":["--- V19/reference-backend","+++ CURRENT/reference-backend","@@ -31,9 +31,9 @@"," [ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"," [ \"$QGA\" = active ] || BAD=$((BAD+1))"," "," if [ \"$BAD\" = 0 ]; then STATUS=OK; COLOR=\"#0a7f38\"; BADGE=\"OK\"; else STATUS=WARN; COLOR=\"#b26a00\"; BADGE=\"REVIEW\"; fi","-STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"","+STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://<PRIVATE_IPV4>:8080/ STATUS_URL=http://<PRIVATE_IPV4>:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""," echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"," echo \"$STATUS_LINE\" > \"$WEB/status.txt\""," "," cat > \"$WEB/index.html\" <<HTML","@@ -55,9 +55,9 @@"," </style>"," </head>"," <body>"," <h1>Homelab Cluster Admin</h1>","-<div class=\"small\">generated UTC: $TS · VM180 · [PRIVATE_IP] · refresh 60s</div>","+<div class=\"small\">generated UTC: $TS · VM180 · <PRIVATE_IPV4> · refresh 60s</div>"," <p><span class=\"badge\">$BADGE</span></p>"," "," <div class=\"grid\">"," <div class=\"card\"><h2>Self</h2><pre>$SELF"],"published_lines":24,"total_lines":24,"truncated":false},"remote_identity":{"euid":0,"hostname":"cluster-admin"},"rollback_restored":false,"rollback_started":false,"schema":22,"status":"OK","transport":{"rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":35786,"stdout_sha256":"cf04063175c8c32ec5f20bb9377e91c2173611df31997715338fcd4f7af95681"},"units":[{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:20:15 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:20:15 UTC","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:15 UTC] ; stop_time=[Wed 2026-08-05 14:20:15 UTC] ; pid=2016430 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","ExecMainExitTimestamp":"","ExecMainStartTimestamp":"Tue 2026-07-28 11:06:07 UTC","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:20:09 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:20:08 UTC","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:08 UTC] ; stop_time=[Wed 2026-08-05 14:20:09 UTC] ; pid=2016411 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:15:31 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:15:25 UTC","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:31 UTC] ; pid=2015620 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:15:25 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:15:25 UTC","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:25 UTC] ; pid=2015621 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:15:25 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:15:25 UTC","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:15:25 UTC] ; stop_time=[Wed 2026-08-05 14:15:25 UTC] ; pid=2015622 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","ExecMainExitTimestamp":"Wed 2026-08-05 14:20:15 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:20:15 UTC","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:20:15 UTC] ; stop_time=[Wed 2026-08-05 14:20:15 UTC] ; pid=2016431 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","ExecMainExitTimestamp":"","ExecMainStartTimestamp":"Tue 2026-07-28 11:06:03 UTC","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"failed","ExecMainExitTimestamp":"Wed 2026-08-05 14:11:28 UTC","ExecMainStartTimestamp":"Wed 2026-08-05 14:11:28 UTC","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:11:28 UTC] ; stop_time=[Wed 2026-08-05 14:11:28 UTC] ; pid=2015065 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","LoadState":"loaded","Result":"exit-code","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027\n"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target\n"}],"v19_exact_result":{"changes_made":false,"control_plane_error":null,"mutation_outcome":"NO_MUTATION","output_bytes":79419,"output_sha256":"136221f755ac0a721469151448174aef1afce2fb7597afcd8ba4652284099fc9","output_source":"runner_json","rollback_restored":null,"rollback_started":false,"runner_document_rc":0,"runner_json_sha256":"bc3ce708ad879590b3850b11dd8282ab9cc2faa48274b8dfd3c6a486bd3e1360","runner_rc":0,"runner_status":"OK","runner_text_sha256":"5bab552a2de42048d35e4e73b12267670153bda3b91e14ae8731163b03969f51","runner_wrapper_matches_document":true,"state_document_sha256":"ba2860a7aab047d7f38ca1141d6f2971e61e990cff504a1f28dce08464c4ecf7","state_path_sha256":"f0448a841353b3dc9042f881326d555faaa928689d01b95b4259170feaccbeda","state_status":"OK"},"v21_exact_result":{"changes_made":false,"control_plane_error":null,"exact_error":"remote python failed: Traceback (most recent call last):\n  File \"<stdin>\", line 14, in <module>\nRuntimeError: sha drift /var/www/homelab-cluster-admin/index.html\n","exact_error_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":843,"output_sha256":"dcd6c5631069f8d522b62ac544e52af1fd03f187bb862ecfa240fdf6b628b199","output_source":"runner_json","rollback_restored":false,"rollback_started":false,"runner_document_rc":1,"runner_json_sha256":"3ad8f45bdaff6a3b68a44a0c1bb22241dcd39e115728837ae5594b292321898e","runner_rc":1,"runner_status":"FAIL","runner_text_sha256":"78d8d650581eafcde8a7ab1498fc2269bb3c4b3b46ca0e388401f3db88164771","runner_wrapper_matches_document":true,"state_document_sha256":"555ae2a4e44cf6d6dac5216dfd5a8f58d4fdcf667392c3c03d3f79f099655fca","state_path_sha256":"1e2f176c02f8bd3d63a101110406e855e05023596ee5f05a49ee4356d8c84f8a","state_status":"FAIL"}}
 
 OUTPUT_END
 CHAT_OUTPUT_END