homelab-public-context/runtime/latest.json

{
  "schema_version": 1,
  "channel": "homelab-runtime",
  "command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z",
  "status": "OK",
  "rc": 0,
  "host": "pve01",
  "mode": "verify",
  "component": "homelab-cluster-admin-update-visibility",
  "started_at_utc": "2026-08-05T14:36:11Z",
  "finished_at_utc": "2026-08-05T14:36:17Z",
  "reference_register_checked": true,
  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
  "error_register_checked": true,
  "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
  "command_sha256": "d8613dc60bb222bd08ec67d6351af6f25843e63459cfa7dd0b0e0efb647b18ff",
  "duplicate_failed_command_blocked": false,
  "block_reason": null,
  "execution_started": true,
  "change_declared": false,
  "result_contract_valid": true,
  "result_contract_status": null,
  "result_contract_error": null,
  "command_rc": 0,
  "changes_made": false,
  "rollback_started": false,
  "rollback_restored": null,
  "mutation_outcome": "NO_MUTATION",
  "sanitized": true,
  "secrets_included": false,
  "private_addresses_included": false,
  "raw_evidence_retained_locally": true,
  "raw_evidence_sha256": "7d4a33d0578aab214f4377ead7188d851c0c23e436a92a798b62ffd63eff3e8e",
  "sanitized_output_sha256": "48cefb450e5f72ed68ed1bf09bbbc4015c82244cd46447d02720a5f3c822ee45",
  "output_truncated_in_json": false,
  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z\",\"command_rc\":0,\"control_plane_error\":null,\"exact_unit_count\":1,\"final_strategy\":\"live-baseline-late-read-with-atomic-recheck\",\"index_sha256_first\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"index_sha256_second\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"index_stable_over_3_seconds\":true,\"matching_source_count\":4,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_final_transaction\":true,\"remote_probe_root\":true,\"report_bytes\":27858,\"report_sha256\":\"76ff00a1e7d991d514a37a438f5107afb49f2713780f5fa33f25860bc8971e93\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"strong_candidate_count\":0,\"unique_producer_found\":false,\"unique_producer_path\":null,\"unique_producer_sha256\":null,\"v23_exact_result_identity_confirmed\":true,\"v23_exact_sha_drift_confirmed\":true,\"version\":1}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V24-INDEX-PRODUCER-READ-ONLY-20260805T143000Z\",\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"remote_producer_analysis\":{\"candidate_units\":[\"homelab-cluster-admin-webpanel-render.service\",\"homelab-cluster-admin-webpanel.service\"],\"euid\":0,\"exact_units\":[{\"cat_rc\":0,\"directory_reference\":true,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"}],\"gate\":{\"next_required_action\":\"use live-baseline same-transaction patch with regeneration guard\",\"ready_for_generator_patch_design\":false,\"strong_candidate_count\":0,\"unique_producer_source\":null},\"hostname\":\"cluster-admin\",\"index\":{\"bytes\":10353,\"ctime_ns\":1785940538091660296,\"gid\":0,\"inode\":135903,\"mode\":\"0o644\",\"mtime_ns\":1785940538091660296,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256_first\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"sha256_second\":\"eca85a4d2f2902a5b2f3dda71ec5204f1864e9742d5528a98faa98696624495a\",\"stable_over_3_seconds\":true,\"title_lines\":[\"<title>Homelab Cluster Admin</title>\",\"<h1>Homelab Cluster Admin</h1>\",\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\",\"<div class=<REDACTED>\",\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\",\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:30:59Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:00Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\",\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T14:31:02Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\",\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T14:31:05Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\",\"OK severity=<REDACTED>\",\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\",\"STATUS=WARN TS=2026-08-05T14:30:58Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"],\"uid\":0},\"journal\":{\"homelab-cluster-admin-webpanel-render.service\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"Aug 05 14:33:28 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\\nAug 05 14:33:28 cluster-admin render-webpanel-cycle-break.sh[2018935]: STATUS=WARN TS=2026-08-05T14:33:28Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\nAug 05 14:33:28 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\\nAug 05 14:33:28 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\\nAug 05 14:34:37 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\\nAug 05 14:34:37 cluster-admin render-webpanel-cycle-break.sh[2019068]: STATUS=WARN TS=2026-08-05T14:34:37Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\nAug 05 14:34:37 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\\nAug 05 14:34:37 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\\nAug 05 14:35:38 cluster-admin systemd[1]: Starting homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel...\\nAug 05 14:35:38 cluster-admin render-webpanel-cycle-break.sh[2019204]: STATUS=WARN TS=2026-08-05T14:35:38Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\nAug 05 14:35:38 cluster-admin systemd[1]: homelab-cluster-admin-webpanel-render.service: Deactivated successfully.\\nAug 05 14:35:38 cluster-admin systemd[1]: Finished homelab-cluster-admin-webpanel-render.service - Render Homelab cluster-admin web panel.\\n\"},\"homelab-cluster-admin-webpanel.service\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"Aug 05 14:33:07 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:33:07] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:04 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:04] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:08 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:08] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:23 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:23] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\nAug 05 14:34:38 cluster-admin python3[439]: [PRIVATE_IP] - - [05/Aug/2026 14:34:38] \\\"HEAD /status.txt HTTP/1.1\\\" 200 -\\n\"}},\"matches\":[{\"bytes\":1433,\"contexts\":[{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"score\":80,\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\",\"write_logic\":true},{\"bytes\":3708,\"contexts\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":39,\"text\":\"cat > \\\"$WEB/index.html\\\" <<HTML\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"score\":80,\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"write_logic\":true},{\"bytes\":301,\"contexts\":[{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"score\":50,\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"write_logic\":false},{\"bytes\":301,\"contexts\":[{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"directory_path\":true,\"exact_path\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"score\":50,\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"write_logic\":false}],\"scan\":{\"directories_scanned\":65,\"files_scanned\":387,\"matching_files\":4},\"units\":[{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019159 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:35:38 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:47 UTC] ; stop_time=[Wed 2026-08-05 14:35:48 UTC] ; pid=2019217 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:35:47 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:14 UTC] ; pid=2019263 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:36:08 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019264 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:36:08 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:36:08 UTC] ; stop_time=[Wed 2026-08-05 14:36:08 UTC] ; pid=2019265 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:36:08 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:35:38 UTC] ; stop_time=[Wed 2026-08-05 14:35:38 UTC] ; pid=2019160 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:35:38 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"directory_reference\":true,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"LoadState\":\"loaded\",\"Result\":\"success\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 14:26:28 UTC] ; stop_time=[Wed 2026-08-05 14:26:28 UTC] ; pid=2017794 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"LoadState\":\"loaded\",\"Result\":\"exit-code\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"directory_reference\":false,\"exact_index_reference\":false,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LastTriggerUSec\":\"Wed 2026-08-05 14:26:28 UTC\",\"LoadState\":\"loaded\",\"NextElapseUSecRealtime\":\"\",\"Result\":\"success\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}]},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":24,\"status\":\"OK\",\"transaction_gate\":{\"index_stable_over_sample\":true,\"ready_for_final_transaction\":true,\"strategy\":\"live-baseline-late-read-with-atomic-recheck\",\"strong_candidate_count\":0,\"unique_producer_found\":false,\"unique_producer_path\":null,\"unique_producer_sha256\":null},\"transport\":{\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":26124,\"stdout_sha256\":\"9694e9ab7966de769d9439dcd24d21cacb20a8a3a548c48d85544aad645d63f6\"},\"v23_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"exact_error\":\"remote python failed: Traceback (most recent call last):\\n  File \\\"<stdin>\\\", line 14, in <module>\\nRuntimeError: sha drift /var/www/homelab-cluster-admin/index.html\\n\",\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":843,\"output_sha256\":\"1bfd49a2a2f0b5f0ed1e2b386c132b63c6fcd7b7bb17b50e301783e803d0bc0c\",\"output_source\":\"runner_json\",\"rollback_restored\":false,\"rollback_started\":false,\"runner_document_rc\":1,\"runner_json_sha256\":\"eabc1c56bfd307512e2914d2163ab7e8accd5ea56adec9d2ae804e305c1c6fbe\",\"runner_rc\":1,\"runner_status\":\"FAIL\",\"runner_text_sha256\":\"677c6def8ad8cff8d01cec0c743d09978c1924f962fb9cc931472cad6f3d038a\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"16c1be27a9ba7efa83b40e4977f2b3cf83e6150dd809febc8a452662ed20dc20\",\"state_path_sha256\":\"2d957c8685cec0699340791b7bea723cb0cfd552a707eac454ec2d8aac23b87e\",\"state_status\":\"FAIL\"}}\n"
}