homelab-public-context/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z.txt

CHAT_OUTPUT_BEGIN
COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z
STATUS=OK
RC=0
HOST=pve01
MODE=verify
COMPONENT=homelab-cluster-admin-update-visibility
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
COMMAND_SHA256=d8613dc60bb222bd08ec67d6351af6f25843e63459cfa7dd0b0e0efb647b18ff
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=7d4a33d0578aab214f4377ead7188d851c0c23e436a92a798b62ffd63eff3e8e
SANITIZED_OUTPUT_SHA256=48cefb450e5f72ed68ed1bf09bbbc4015c82244cd46447d02720a5f3c822ee45
OUTPUT_BEGIN
HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z","command_rc":0,"control_plane_error":null,"exact_unit_count":1,"final_strategy":"live-baseline-late-read-with-atomic-recheck","index_sha256_first":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","index_sha256_second":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","index_stable_over_3_seconds":true,"matching_source_count":4,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_final_transaction":true,"remote_probe_root":true,"report_bytes":27858,"report_sha256":"76ff00a1e7d991d514a37a438f5107afb49f2713780f5fa33f25860bc8971e93","rollback_restored":false,"rollback_started":false,"status":"OK","strong_candidate_count":0,"unique_producer_found":false,"unique_producer_path":null,"unique_producer_sha256":null,"v23_exact_result_identity_confirmed":true,"v23_exact_sha_drift_confirmed":true,"version":1}
{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z","mutation_outcome":"NO_MUTATION","read_only":true,"remote_producer_analysis":{"candidate_units":["homelab-cluster-admin-webpanel-render.service","homelab-cluster-admin-webpanel.service"],"euid":0,"exact_units":[{"cat_rc":0,"directory_reference":true,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"}],"gate":{"next_required_action":"use live-baseline same-transaction patch with regeneration guard","ready_for_generator_patch_design":false,"strong_candidate_count":0,"unique_producer_source":null},"hostname":"cluster-admin","index":{"bytes":10353,"ctime_ns":1785940538091660296,"gid":0,"inode":135903,"mode":"0o644","mtime_ns":1785940538091660296,"path":"/var/www/homelab-cluster-admin/index.html","sha256_first":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","sha256_second":"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a","stable_over_3_seconds":true,"title_lines":["<title>Homelab Cluster Admin</title>","<h1>Homelab Cluster Admin</h1>","<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>","<div class=<REDACTED>","<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>","<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>","<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer","pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:30:59Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:00Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2","pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:31:02Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0","edge ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:05Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active","OK severity=<REDACTED>","OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0","STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"],"uid":0},"journal":{"homelab-cluster-admin-webpanel-render.service":{"rc":0,"stderr":"","stdout":"Aug 05 14:33:28 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\nAug 05 14:33:28 cluster-admin render-webpanel-cycle-break.sh[2018935]: STATUS=WARN TS=2026-08-05T14:33:28Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\nAug 05 14:33:28 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\nAug 05 14:33:28 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\nAug 05 14:34:37 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\nAug 05 14:34:37 cluster-admin render-webpanel-cycle-break.sh[2019068]: STATUS=WARN TS=2026-08-05T14:34:37Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\nAug 05 14:34:37 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\nAug 05 14:34:37 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\nAug 05 14:35:38 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\nAug 05 14:35:38 cluster-admin render-webpanel-cycle-break.sh[2019204]: STATUS=WARN TS=2026-08-05T14:35:38Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\nAug 05 14:35:38 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\nAug 05 14:35:38 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\n"},"homelab-cluster-admin-webpanel.service":{"rc":0,"stderr":"","stdout":"Aug 05 14:33:07 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:33:07] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:04 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:04] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:08 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:08] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:23 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:23] \"HEAD /status.txt HTTP/1.1\" 200 -\nAug 05 14:34:38 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:38] \"HEAD /status.txt HTTP/1.1\" 200 -\n"}},"matches":[{"bytes":1433,"contexts":[{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"directory_path":true,"exact_path":false,"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","score":80,"sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792","write_logic":true},{"bytes":3708,"contexts":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":39,"text":"cat > \"$WEB/index.html\" <<HTML"}],"directory_path":true,"exact_path":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","score":80,"sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","write_logic":true},{"bytes":301,"contexts":[{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"directory_path":true,"exact_path":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","score":50,"sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","write_logic":false},{"bytes":301,"contexts":[{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"directory_path":true,"exact_path":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","score":50,"sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","write_logic":false}],"scan":{"directories_scanned":65,"files_scanned":387,"matching_files":4},"units":[{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019159 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LastTriggerUSec":"Wed 2026-08-05 14:35:38 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:47 UTC] ; stop_time=[Wed 2026-08-05 14:35:48 UTC] ; pid=2019217 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LastTriggerUSec":"Wed 2026-08-05 14:35:47 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:14 UTC] ; pid=2019263 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LastTriggerUSec":"Wed 2026-08-05 14:36:08 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019264 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LastTriggerUSec":"Wed 2026-08-05 14:36:08 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019265 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LastTriggerUSec":"Wed 2026-08-05 14:36:08 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019160 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","LoadState":"loaded","Result":"success","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LastTriggerUSec":"Wed 2026-08-05 14:35:38 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"directory_reference":true,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","LoadState":"loaded","Result":"success","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:26:28 UTC] ; stop_time=[Wed 2026-08-05 14:26:28 UTC] ; pid=2017794 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","LoadState":"loaded","Result":"exit-code","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"directory_reference":false,"exact_index_reference":false,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LastTriggerUSec":"Wed 2026-08-05 14:26:28 UTC","LoadState":"loaded","NextElapseUSecRealtime":"","Result":"success","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}]},"rollback_restored":false,"rollback_started":false,"schema":24,"status":"OK","transaction_gate":{"index_stable_over_sample":true,"ready_for_final_transaction":true,"strategy":"live-baseline-late-read-with-atomic-recheck","strong_candidate_count":0,"unique_producer_found":false,"unique_producer_path":null,"unique_producer_sha256":null},"transport":{"rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":26124,"stdout_sha256":"9694e9ab7966de769d9439dcd24d21cacb20a8a3a548c48d85544aad645d63f6"},"v23_exact_result":{"changes_made":false,"control_plane_error":null,"exact_error":"remote python failed: Traceback (most recent call last):\n  File \"<stdin>\", line 14, in <module>\nRuntimeError: sha drift /var/www/homelab-cluster-admin/index.html\n","identity_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":843,"output_sha256":"1bfd49a2a2f0b5f0ed1e2b386c132b63c6fcd7b7bb17b50e301783e803d0bc0c","output_source":"runner_json","rollback_restored":false,"rollback_started":false,"runner_document_rc":1,"runner_json_sha256":"eabc1c56bfd307512e2914d2163ab7e8accd5ea56adec9d2ae804e305c1c6fbe","runner_rc":1,"runner_status":"FAIL","runner_text_sha256":"677c6def8ad8cff8d01cec0c743d09978c1924f962fb9cc931472cad6f3d038a","runner_wrapper_matches_document":true,"state_document_sha256":"16c1be27a9ba7efa83b40e4977f2b3cf83e6150dd809febc8a452662ed20dc20","state_path_sha256":"2d957c8685cec0699340791b7bea723cb0cfd552a707eac454ec2d8aac23b87e","state_status":"FAIL"}}

OUTPUT_END
CHAT_OUTPUT_END