homelab-public-context/runtime/latest.txt

CHAT_OUTPUT_BEGIN
COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V19-REMOTE-CORE-READ-ONLY-20260805T133000Z
STATUS=OK
RC=0
HOST=pve01
MODE=verify
COMPONENT=homelab-cluster-admin-update-visibility
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
COMMAND_SHA256=ed901eae70df3049b86d3644e4a1166315c959eff5c2a64715b75ce544b7abf4
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=136221f755ac0a721469151448174aef1afce2fb7597afcd8ba4652284099fc9
SANITIZED_OUTPUT_SHA256=136221f755ac0a721469151448174aef1afce2fb7597afcd8ba4652284099fc9
OUTPUT_BEGIN
HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V19-REMOTE-CORE-READ-ONLY-20260805T133000Z","command_rc":0,"control_plane_error":null,"frontend_index_count":1,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_final_atomic_design":true,"reference_backend_count":1,"relevant_unit_count":16,"remote_probe_root":true,"report_bytes":78679,"report_sha256":"82e69b5c37c04ad98c95899686cd170fa289007b289de00cede181014256ec87","rollback_restored":false,"rollback_started":false,"selected_source_count":18,"status":"OK","v18_exact_result_identity_confirmed":true,"v18_output_sha256":"5b17a087bd51a9f4e901cd820e129160e70081636d2da63d902f481305df4f99","v18_runner_wrapper_matches_document":false,"version":1}
{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V19-REMOTE-CORE-READ-ONLY-20260805T133000Z","mutation_outcome":"NO_MUTATION","next_gate":{"next_required_action":"build one corrected atomic local+remote transaction","ready_for_final_atomic_design":true},"read_only":true,"remote_core":{"exact_preconditions":{"/etc/systemd/system/cluster-admin-incident-engine-collector.service":{"bytes":616,"mode":"0o644","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},"/etc/systemd/system/cluster-admin-incident-engine-collector.timer":{"bytes":231,"mode":"0o644","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},"/etc/systemd/system/cluster-admin-incident-engine.service":{"bytes":803,"mode":"0o644","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},"/etc/systemd/system/cluster-admin-probe-agent.service":{"bytes":1190,"mode":"0o644","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},"/etc/systemd/system/cluster-admin-probe-agent.timer":{"bytes":223,"mode":"0o644","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},"/etc/systemd/system/homelab-cluster-admin-full-observer.service":{"bytes":133,"mode":"0o644","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},"/etc/systemd/system/homelab-cluster-admin-full-observer.timer":{"bytes":154,"mode":"0o644","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},"/etc/systemd/system/homelab-cluster-admin-observer.service":{"bytes":131,"mode":"0o644","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},"/etc/systemd/system/homelab-cluster-admin-observer.timer":{"bytes":149,"mode":"0o644","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},"/etc/systemd/system/homelab-cluster-admin-self-check.service":{"bytes":127,"mode":"0o644","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},"/etc/systemd/system/homelab-cluster-admin-self-check.timer":{"bytes":151,"mode":"0o644","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service":{"bytes":138,"mode":"0o644","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer":{"bytes":154,"mode":"0o644","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},"/etc/systemd/system/homelab-cluster-admin-webpanel.service":{"bytes":301,"mode":"0o644","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},"/etc/systemd/system/homelab-stage4i-verify.service":{"bytes":465,"mode":"0o644","sha256":"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b"},"/etc/systemd/system/homelab-stage4i-verify.timer":{"bytes":224,"mode":"0o644","sha256":"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396"},"/opt/homelab-cluster-admin/cluster-observer.sh":{"bytes":2832,"mode":"0o755","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},"/opt/homelab-cluster-admin/full-observer.sh":{"bytes":3471,"mode":"0o755","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh":{"bytes":1433,"mode":"0o750","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},"/opt/homelab-cluster-admin/render-webpanel.sh":{"bytes":3708,"mode":"0o755","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},"/opt/homelab-cluster-admin/self-check.sh":{"bytes":848,"mode":"0o755","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},"/usr/local/sbin/homelab-stage4i-verify":{"bytes":2541,"mode":"0o750","sha256":"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d"},"/var/www/homelab-cluster-admin/index.html":{"bytes":10353,"mode":"0o644","sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89"}},"exec_paths":["/opt/homelab-cluster-admin/cluster-observer.sh","/opt/homelab-cluster-admin/full-observer.sh","/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","/opt/homelab-cluster-admin/self-check.sh","/usr/bin/python3","/usr/local/sbin/homelab-stage4i-verify"],"fragment_paths":["/etc/systemd/system/cluster-admin-incident-engine-collector.service","/etc/systemd/system/cluster-admin-incident-engine-collector.timer","/etc/systemd/system/cluster-admin-incident-engine.service","/etc/systemd/system/cluster-admin-probe-agent.service","/etc/systemd/system/cluster-admin-probe-agent.timer","/etc/systemd/system/homelab-cluster-admin-full-observer.service","/etc/systemd/system/homelab-cluster-admin-full-observer.timer","/etc/systemd/system/homelab-cluster-admin-observer.service","/etc/systemd/system/homelab-cluster-admin-observer.timer","/etc/systemd/system/homelab-cluster-admin-self-check.service","/etc/systemd/system/homelab-cluster-admin-self-check.timer","/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","/etc/systemd/system/homelab-cluster-admin-webpanel.service","/etc/systemd/system/homelab-stage4i-verify.service","/etc/systemd/system/homelab-stage4i-verify.timer"],"gate":{"exact_source_count":30,"index_source_count":1,"ready_for_atomic_design":true,"reference_source_count":1,"unit_record_count":16},"index_path":"/var/www/homelab-cluster-admin/index.html","reference_path":"/opt/homelab-cluster-admin/render-webpanel.sh","relevant_unit_count":16,"relevant_units":[{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006178 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","Group":"clusteradmin","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],<PRIVATE_IPV4>\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:27 UTC] ; stop_time=[Wed 2026-08-05 13:20:28 UTC] ; pid=2006164 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:15 UTC] ; pid=2005776 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005777 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005778 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006179 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","Group":"","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:11:27 UTC] ; stop_time=[Wed 2026-08-05 13:11:27 UTC] ; pid=2004608 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","Group":"root","LoadState":"loaded","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","User":"root","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}],"remote_euid":0,"remote_hostname":"cluster-admin","selected_source_count":18,"selected_sources":[{"anchors":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":39,"text":"cat > \"$WEB/index.html\" <<HTML"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"binary":null,"bytes":3708,"line_count":79,"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","priority":100,"role":"reference-backend","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":5,"text":"mkdir -p \"$WEB\""},{"line":6,"text":"BAD=0"},{"line":7,"text":""},{"line":8,"text":"line_or_missing() {"},{"line":9,"text":"  f=\"$1\""},{"line":10,"text":"  if [ -s \"$f\" ]; then head -n1 \"$f\"; else echo \"STATUS=WARN TYPE=missing FILE=$f\"; fi"},{"line":11,"text":"}"},{"line":12,"text":""},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":20,"text":"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":21,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":22,"text":""},{"line":23,"text":"for line in \"$SELF\" \"$OBSERVER\" \"$FULL\" \"$BACKUP\"; do"},{"line":24,"text":"  echo \"$line\" | grep -q '^STATUS=OK' || BAD=$((BAD+1))"},{"line":25,"text":"done"},{"line":26,"text":"echo \"$FULL\" | grep -q 'BAD=0' || BAD=$((BAD+1))"},{"line":27,"text":"echo \"$FULL\" | grep -q 'WARN=0' || BAD=$((BAD+1))"},{"line":28,"text":"echo \"$BACKUP\" | grep -q 'BAD=0' || BAD=$((BAD+1))"},{"line":29,"text":"echo \"$BACKUP\" | grep -q 'MAIL_CLOUD_2COPY_VERIFY_OK' || BAD=$((BAD+1))"},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":31,"text":"[ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":32,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":33,"text":""},{"line":34,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; COLOR=\"#0a7f38\"; BADGE=\"OK\"; else STATUS=WARN; COLOR=\"#b26a00\"; BADGE=\"REVIEW\"; fi"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":37,"text":"echo \"$STATUS_LINE\" > \"$WEB/status.txt\""},{"line":38,"text":""},{"line":39,"text":"cat > \"$WEB/index.html\" <<HTML"},{"line":40,"text":"<!doctype html>"},{"line":41,"text":"<html lang=\"ru\">"},{"line":42,"text":"<head>"},{"line":43,"text":"<meta charset=\"utf-8\">"},{"line":44,"text":"<meta http-equiv=\"refresh\" content=\"60\">"},{"line":45,"text":"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":47,"text":"<style>"},{"line":48,"text":"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}"},{"line":49,"text":".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}"},{"line":50,"text":"h1{margin:0 0 8px}"},{"line":51,"text":".badge{display:inline-block;background:$COLOR;color:white;padding:8px 14px;border-radius:999px;font-weight:700}"},{"line":52,"text":"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}"},{"line":53,"text":".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}"},{"line":54,"text":".small{color:#aab2c0}"},{"line":55,"text":"</style>"},{"line":56,"text":"</head>"},{"line":57,"text":"<body>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":59,"text":"<div class=\"small\">generated UTC: $TS · VM180 · [PRIVATE_IP] · refresh 60s</div>"},{"line":60,"text":"<p><span class=\"badge\">$BADGE</span></p>"},{"line":61,"text":""},{"line":62,"text":"<div class=\"grid\">"},{"line":63,"text":"<div class=\"card\"><h2>Self</h2><pre>$SELF"},{"line":64,"text":"FAILED_UNITS=$FAILED"},{"line":65,"text":"QGA=$QGA</pre></div>"},{"line":66,"text":"<div class=\"card\"><h2>Observer</h2><pre>$OBSERVER</pre></div>"},{"line":67,"text":"<div class=\"card\"><h2>Full observer</h2><pre>$FULL</pre></div>"},{"line":68,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>$BACKUP</pre></div>"},{"line":69,"text":"</div>"},{"line":70,"text":""},{"line":71,"text":"<div class=\"card\"><h2>Overall</h2><pre>$OVERALL</pre></div>"},{"line":72,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>$FINAL</pre></div>"},{"line":73,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>$STRICT</pre></div>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":75,"text":"</body>"},{"line":76,"text":"</html>"},{"line":77,"text":"HTML"},{"line":78,"text":""},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}]},{"anchors":[{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":45,"text":"## Service map checks"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"}],"binary":null,"bytes":10353,"line_count":80,"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","priority":95,"role":"frontend-index","sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","source":[{"line":1,"text":"<!doctype html>"},{"line":2,"text":"<html lang=\"ru\">"},{"line":3,"text":"<head>"},{"line":4,"text":"<meta charset=\"utf-8\">"},{"line":5,"text":"<meta http-equiv=\"refresh\" content=\"60\">"},{"line":6,"text":"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"},{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":8,"text":"<style>"},{"line":9,"text":"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}"},{"line":10,"text":".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}"},{"line":11,"text":"h1{margin:0 0 8px}"},{"line":12,"text":".badge{display:inline-block;background:#b26a00;color:white;padding:8px 14px;border-radius:999px;font-weight:700}"},{"line":13,"text":"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}"},{"line":14,"text":".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}"},{"line":15,"text":".small{color:#aab2c0}"},{"line":16,"text":"</style>"},{"line":17,"text":"</head>"},{"line":18,"text":"<body>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":20,"text":"<div class=\"small\">generated UTC: 2026-08-05T13:20:48Z · VM180 · [PRIVATE_IP] · refresh 60s</div>"},{"line":21,"text":"<p><span class=\"badge\">REVIEW</span></p>"},{"line":22,"text":""},{"line":23,"text":"<div class=\"grid\">"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":25,"text":"FAILED_UNITS=1"},{"line":26,"text":"QGA=active</pre></div>"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":30,"text":"</div>"},{"line":31,"text":""},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":36,"text":"generated_utc=2026-08-05T13:19:08Z"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":38,"text":""},{"line":39,"text":"## Restricted probes"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":44,"text":""},{"line":45,"text":"## Service map checks"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":47,"text":"OK severity=critical kind=tcp name=pve01_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":48,"text":"OK severity=critical kind=tcp name=pve01_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0"},{"line":49,"text":"OK severity=critical kind=tcp name=pve01_health target=[PRIVATE_IP] value=9101 tcp_9101_rc=0"},{"line":50,"text":"OK severity=critical kind=tcp name=pve02_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":51,"text":"OK severity=critical kind=tcp name=pve02_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0"},{"line":52,"text":"OK severity=critical kind=tcp name=pve03_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":53,"text":"OK severity=critical kind=tcp name=pve03_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0"},{"line":54,"text":"OK severity=critical kind=tcp name=dns1_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0"},{"line":55,"text":"OK severity=critical kind=tcp name=dns2_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0"},{"line":56,"text":"OK severity=critical kind=tcp name=edge_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":58,"text":"OK severity=critical kind=ping name=vm130_edge_vm_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":59,"text":"OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":60,"text":"OK severity=critical kind=ping name=vm150_nextcloud_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":61,"text":"OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":62,"text":"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1"},{"line":63,"text":"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":64,"text":"OK severity=critical kind=ping name=vm160_forum_prod_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":65,"text":"OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":66,"text":"OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=0"},{"line":67,"text":"INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":68,"text":"OK severity=critical kind=ping name=vm170_core_apps_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":69,"text":"OK severity=critical kind=tcp name=vm170_core_apps_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":70,"text":"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1"},{"line":71,"text":"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":72,"text":"OK severity=critical kind=ping name=vm171_monitoring_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":73,"text":"OK severity=critical kind=tcp name=vm171_monitoring_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":74,"text":"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1"},{"line":75,"text":"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":76,"text":""},{"line":77,"text":"## Result"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":79,"text":"</body>"},{"line":80,"text":"</html>"}]},{"anchors":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"binary":null,"bytes":2832,"line_count":96,"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","priority":92,"role":"exec-target","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":4,"text":"exec 9>\"$LOCK\""},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":6,"text":""},{"line":7,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":10,"text":"REPORT=\"$ROOT/latest.md\""},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":12,"text":"TMP=\"$REPORT.tmp\""},{"line":13,"text":"mkdir -p \"$ROOT\" \"$HEALTH_DIR\""},{"line":14,"text":"BAD=0"},{"line":15,"text":"WARN=0"},{"line":16,"text":""},{"line":17,"text":"check_ping() {"},{"line":18,"text":"  name=\"$1\"; ip=\"$2\""},{"line":19,"text":"  ping -c1 -W1 \"$ip\" >/dev/null 2>&1"},{"line":20,"text":"  rc=$?"},{"line":21,"text":"  echo \"$name ping rc=$rc\" >> \"$TMP\""},{"line":22,"text":"  [ \"$rc\" = 0 ] || BAD=$((BAD+1))"},{"line":23,"text":"}"},{"line":24,"text":""},{"line":25,"text":"check_tcp() {"},{"line":26,"text":"  name=\"$1\"; ip=\"$2\"; port=\"$3\""},{"line":27,"text":"  timeout 2 bash -c \"</dev/tcp/$ip/$port\" >/dev/null 2>&1"},{"line":28,"text":"  rc=$?"},{"line":29,"text":"  echo \"$name tcp/$port rc=$rc\" >> \"$TMP\""},{"line":30,"text":"  [ \"$rc\" = 0 ] || BAD=$((BAD+1))"},{"line":31,"text":"}"},{"line":32,"text":""},{"line":33,"text":"check_http_status_ok() {"},{"line":34,"text":"  name=\"$1\"; url=\"$2\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":36,"text":"  rc=$?"},{"line":37,"text":"  echo \"$name http rc=$rc line=$line\" >> \"$TMP\""},{"line":38,"text":"  echo \"$line\" | grep -q \"^STATUS=\""},{"line":39,"text":"  okrc=$?"},{"line":40,"text":"  [ \"$rc\" = 0 ] && [ \"$okrc\" = 0 ] || BAD=$((BAD+1))"},{"line":41,"text":"}"},{"line":42,"text":""},{"line":43,"text":"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":44,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":46,"text":""},{"line":47,"text":"{"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":49,"text":"  echo \"generated_utc=$TS\""},{"line":50,"text":"  echo"},{"line":51,"text":"  echo \"## Self\""},{"line":52,"text":"  echo \"host=$(hostname 2>/dev/null)\""},{"line":53,"text":"  echo \"failed_units=$FAILED\""},{"line":54,"text":"  echo \"qga=$QGA\""},{"line":55,"text":"  echo \"self_check=$SELF_LINE\""},{"line":56,"text":"  echo"},{"line":57,"text":"  echo \"## Network checks\""},{"line":58,"text":"} > \"$TMP\""},{"line":59,"text":""},{"line":60,"text":"[ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":61,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":62,"text":"echo \"$SELF_LINE\" | grep -q \"^STATUS=OK\" || BAD=$((BAD+1))"},{"line":63,"text":""},{"line":64,"text":"check_ping pve01 [PRIVATE_IP]"},{"line":65,"text":"check_tcp pve01_ssh [PRIVATE_IP] 22"},{"line":66,"text":"check_tcp pve01_proxmox [PRIVATE_IP] 8006"},{"line":67,"text":"check_tcp pve01_health [PRIVATE_IP] 9101"},{"line":68,"text":""},{"line":69,"text":"check_ping pve02 [PRIVATE_IP]"},{"line":70,"text":"check_tcp pve02_ssh [PRIVATE_IP] 22"},{"line":71,"text":"check_tcp pve02_proxmox [PRIVATE_IP] 8006"},{"line":72,"text":""},{"line":73,"text":"check_ping pve03 [PRIVATE_IP]"},{"line":74,"text":"check_tcp pve03_ssh [PRIVATE_IP] 22"},{"line":75,"text":"check_tcp pve03_proxmox [PRIVATE_IP] 8006"},{"line":76,"text":""},{"line":77,"text":"check_ping edge [PRIVATE_IP]"},{"line":78,"text":"check_tcp edge_ssh [PRIVATE_IP] 22"},{"line":79,"text":""},{"line":80,"text":"check_ping dns1 [PRIVATE_IP]"},{"line":81,"text":"check_tcp dns1_tcp53 [PRIVATE_IP] 53"},{"line":82,"text":"check_ping dns2 [PRIVATE_IP]"},{"line":83,"text":"check_tcp dns2_tcp53 [PRIVATE_IP] 53"},{"line":84,"text":""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":86,"text":""},{"line":87,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""},{"line":89,"text":"echo \"$LINE\" | tee \"$HEALTH\" >/dev/null"},{"line":90,"text":"{"},{"line":91,"text":"  echo"},{"line":92,"text":"  echo \"## Result\""},{"line":93,"text":"  echo \"$LINE\""},{"line":94,"text":"} >> \"$TMP\""},{"line":95,"text":"mv \"$TMP\" \"$REPORT\""},{"line":96,"text":"cat \"$HEALTH\""}]},{"anchors":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":46,"text":"  echo \"## Service map checks\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"binary":null,"bytes":3471,"line_count":113,"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","priority":92,"role":"exec-target","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":7,"text":"REPORT=\"$ROOT/full-observer-latest.md\""},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":10,"text":"mkdir -p \"$ROOT\" \"$HEALTH_DIR\""},{"line":11,"text":"exec 9>\"$LOCK\""},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":13,"text":""},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":15,"text":"BAD=0"},{"line":16,"text":"WARN=0"},{"line":17,"text":"INFO_OFF=0"},{"line":18,"text":"TOTAL=0"},{"line":19,"text":"CRIT_FAIL=0"},{"line":20,"text":"OPT_OFF=0"},{"line":21,"text":"PROBE_OK=0"},{"line":22,"text":"PROBE_BAD=0"},{"line":23,"text":""},{"line":24,"text":"TMP=\"$REPORT.tmp\""},{"line":25,"text":"{"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":27,"text":"  echo \"generated_utc=$TS\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":29,"text":"  echo"},{"line":30,"text":"  echo \"## Restricted probes\""},{"line":31,"text":"} > \"$TMP\""},{"line":32,"text":""},{"line":33,"text":"for t in \"root@[PRIVATE_IP] pve01\" \"root@[PRIVATE_IP] pve02\" \"root@[PRIVATE_IP] pve03\" \"debian@[PRIVATE_IP] edge\"; do"},{"line":34,"text":"  set -- $t"},{"line":35,"text":"  target=\"$1\"; name=\"$2\""},{"line":36,"text":"  out=$(ssh -n -i \"$K\" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 \"$target\" probe 2>/tmp/full-probe-$name.err)"},{"line":37,"text":"  rc=$?"},{"line":38,"text":"  echo \"$name ssh_rc=$rc $out\" >> \"$TMP\""},{"line":39,"text":"  echo \"$out\" | grep -q '^STATUS=OK'"},{"line":40,"text":"  ok=$?"},{"line":41,"text":"  if [ \"$rc\" = 0 ] && [ \"$ok\" = 0 ]; then PROBE_OK=$((PROBE_OK+1)); else PROBE_BAD=$((PROBE_BAD+1)); BAD=$((BAD+1)); fi"},{"line":42,"text":"done"},{"line":43,"text":""},{"line":44,"text":"{"},{"line":45,"text":"  echo"},{"line":46,"text":"  echo \"## Service map checks\""},{"line":47,"text":"} >> \"$TMP\""},{"line":48,"text":""},{"line":49,"text":"while IFS=\"$(printf '\\t')\" read -r severity kind name target value; do"},{"line":50,"text":"  [ -n \"$severity\" ] || continue"},{"line":51,"text":"  TOTAL=$((TOTAL+1))"},{"line":52,"text":"  rc=0"},{"line":53,"text":"  detail=\"\""},{"line":54,"text":"  case \"$kind\" in"},{"line":55,"text":"    ping)"},{"line":56,"text":"      ping -c1 -W1 \"$target\" >/dev/null 2>&1"},{"line":57,"text":"      rc=$?"},{"line":58,"text":"      detail=\"ping_rc=$rc\""},{"line":59,"text":"      ;;"},{"line":60,"text":"    tcp)"},{"line":61,"text":"      timeout 2 bash -c \"</dev/tcp/$target/$value\" >/dev/null 2>&1"},{"line":62,"text":"      rc=$?"},{"line":63,"text":"      detail=\"tcp_${value}_rc=$rc\""},{"line":64,"text":"      ;;"},{"line":65,"text":"    http_status_ok)"},{"line":66,"text":"      line=$(curl -fsS --max-time 5 \"$value\" 2>/tmp/full-http-$name.err | head -n1)"},{"line":67,"text":"      curlrc=$?"},{"line":68,"text":"      echo \"$line\" | grep -q '^STATUS='"},{"line":69,"text":"      okrc=$?"},{"line":70,"text":"      [ \"$curlrc\" = 0 ] && [ \"$okrc\" = 0 ]"},{"line":71,"text":"      rc=$?"},{"line":72,"text":"      detail=\"http_rc=$curlrc ok_rc=$okrc line=$line\""},{"line":73,"text":"      ;;"},{"line":74,"text":"    missing)"},{"line":75,"text":"      rc=1"},{"line":76,"text":"      detail=\"missing_target=$target value=$value\""},{"line":77,"text":"      ;;"},{"line":78,"text":"    *)"},{"line":79,"text":"      rc=1"},{"line":80,"text":"      detail=\"unknown_kind=$kind\""},{"line":81,"text":"      ;;"},{"line":82,"text":"  esac"},{"line":83,"text":""},{"line":84,"text":"  if [ \"$rc\" = 0 ]; then"},{"line":85,"text":"    echo \"OK severity=$severity kind=$kind name=$name target=$target value=$value $detail\" >> \"$TMP\""},{"line":86,"text":"  else"},{"line":87,"text":"    if [ \"$severity\" = critical ]; then"},{"line":88,"text":"      echo \"FAIL severity=critical kind=$kind name=$name target=$target value=$value $detail\" >> \"$TMP\""},{"line":89,"text":"      CRIT_FAIL=$((CRIT_FAIL+1))"},{"line":90,"text":"      BAD=$((BAD+1))"},{"line":91,"text":"    else"},{"line":92,"text":"      echo \"INFO_OFF severity=optional kind=$kind name=$name target=$target value=$value $detail\" >> \"$TMP\""},{"line":93,"text":"      OPT_OFF=$((OPT_OFF+1))"},{"line":94,"text":"      INFO_OFF=$((INFO_OFF+1))"},{"line":95,"text":"    fi"},{"line":96,"text":"  fi"},{"line":97,"text":"done < \"$MAP\""},{"line":98,"text":""},{"line":99,"text":"SELF_FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":100,"text":"[ \"$SELF_FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":101,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":102,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":103,"text":""},{"line":104,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi"},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""},{"line":106,"text":"echo \"$LINE\" | tee \"$HEALTH\" >/dev/null"},{"line":107,"text":"{"},{"line":108,"text":"  echo"},{"line":109,"text":"  echo \"## Result\""},{"line":110,"text":"  echo \"$LINE\""},{"line":111,"text":"} >> \"$TMP\""},{"line":112,"text":"mv \"$TMP\" \"$REPORT\""},{"line":113,"text":"cat \"$HEALTH\""}]},{"anchors":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"binary":null,"bytes":1433,"line_count":27,"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","priority":92,"role":"exec-target","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set -Eeuo pipefail"},{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"},{"line":6,"text":"ORIGINAL_RC=0"},{"line":7,"text":"\"$ORIGINAL\" || ORIGINAL_RC=$?"},{"line":8,"text":"[ -s \"$HEALTH\" ] || exit \"$ORIGINAL_RC\""},{"line":9,"text":"line=\"$(sed -n \"1p\" \"$HEALTH\" 2>/dev/null || true)\""},{"line":10,"text":"field(){ local key=<REDACTED>"},{"line":11,"text":"self_failed=\"$(field SELF_FAILED_UNITS || true)\""},{"line":12,"text":"qga_state=\"$(field QGA || true)\""},{"line":13,"text":"backup_state=\"$(field BACKUP || true)\""},{"line":14,"text":"if [ \"$ORIGINAL_RC\" -eq 0 ] && [ \"$self_failed\" = \"0\" ] && [ \"$qga_state\" = \"active\" ] && [ \"$backup_state\" = \"MAIL_CLOUD_2COPY_VERIFY_OK\" ]; then"},{"line":15,"text":"  normalized=\"$(printf \"%s\\n\" \"$line\" | sed -E \"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\")\""},{"line":16,"text":"  printf \"%s\\n\" \"$normalized\" > \"$HEALTH\""},{"line":17,"text":"  if [ -f \"$STATUS_FILE\" ]; then"},{"line":18,"text":"    status_line=\"$(sed -n \"1p\" \"$STATUS_FILE\" 2>/dev/null || true)\""},{"line":19,"text":"    status_normalized=\"$(printf \"%s\\n\" \"$status_line\" | sed -E \"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\")\""},{"line":20,"text":"    tmp=\"${STATUS_FILE}.tmp.$$\""},{"line":21,"text":"    { printf \"%s\\n\" \"$status_normalized\"; tail -n +2 \"$STATUS_FILE\" 2>/dev/null || true; } > \"$tmp\""},{"line":22,"text":"    chown --reference=\"$STATUS_FILE\" \"$tmp\" 2>/dev/null || true"},{"line":23,"text":"    chmod --reference=\"$STATUS_FILE\" \"$tmp\" 2>/dev/null || true"},{"line":24,"text":"    mv \"$tmp\" \"$STATUS_FILE\""},{"line":25,"text":"  fi"},{"line":26,"text":"fi"},{"line":27,"text":"exit \"$ORIGINAL_RC\""}]},{"anchors":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"binary":null,"bytes":848,"line_count":17,"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","priority":92,"role":"exec-target","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":4,"text":"BAD=0"},{"line":5,"text":"HOST=$(hostname 2>/dev/null || echo unknown)"},{"line":6,"text":"IP_OK=0"},{"line":7,"text":"ip -4 -br a | grep -q \"[PRIVATE_IP]\" && IP_OK=1 || BAD=$((BAD+1))"},{"line":8,"text":"DNS_OK=0"},{"line":9,"text":"getent hosts pve01 >/dev/null 2>&1 || getent hosts gram1.ru >/dev/null 2>&1"},{"line":10,"text":"[ \"$?\" = 0 ] && DNS_OK=1 || BAD=$((BAD+1))"},{"line":11,"text":"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":12,"text":"[ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":13,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":14,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":15,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi"},{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}]},{"anchors":[{"line":6,"text":"STATE_DIR=\"/var/lib/homelab-cluster-admin/stage4i-verification\""},{"line":17,"text":"        /var/lib/homelab-cluster-admin \\"}],"binary":null,"bytes":2541,"line_count":105,"mode":"0o750","path":"/usr/local/sbin/homelab-stage4i-verify","priority":92,"role":"exec-target","sha256":"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set -euo pipefail"},{"line":3,"text":"export LC_ALL=C"},{"line":4,"text":""},{"line":5,"text":"MARKER=\"CR0019_STAGE4I_IMMUTABLE_V1\""},{"line":6,"text":"STATE_DIR=\"/var/lib/homelab-cluster-admin/stage4i-verification\""},{"line":7,"text":"REPORT=\"$STATE_DIR/latest.env\""},{"line":8,"text":"TMP=\"$(mktemp)\""},{"line":9,"text":""},{"line":10,"text":"cleanup() {"},{"line":11,"text":"    rm -f \"$TMP\""},{"line":12,"text":"}"},{"line":13,"text":"trap cleanup EXIT"},{"line":14,"text":""},{"line":15,"text":"mapfile -t ACTIVATION_FILES < <("},{"line":16,"text":"    find \\"},{"line":17,"text":"        /var/lib/homelab-cluster-admin \\"},{"line":18,"text":"        /opt \\"},{"line":19,"text":"        /srv \\"},{"line":20,"text":"        /usr/local/share \\"},{"line":21,"text":"        -type f \\"},{"line":22,"text":"        -path '*/.stage4i-control/activation.env' \\"},{"line":23,"text":"        -print 2>/dev/null |"},{"line":24,"text":"    sort -u"},{"line":25,"text":")"},{"line":26,"text":""},{"line":27,"text":"if [ \"${#ACTIVATION_FILES[@]}\" -ne 1 ]; then"},{"line":28,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=FAILED\""},{"line":29,"text":"    echo \"REASON=ACTIVATION_FILE_COUNT_${#ACTIVATION_FILES[@]}\""},{"line":30,"text":"    exit 50"},{"line":31,"text":"fi"},{"line":32,"text":""},{"line":33,"text":"ACTIVATION_FILE=\"${ACTIVATION_FILES[0]}\""},{"line":34,"text":"CONTROL_DIR=\"$(dirname \"$ACTIVATION_FILE\")\""},{"line":35,"text":"ACTIVE_LINK=\"$CONTROL_DIR/active\""},{"line":36,"text":"ROLLBACK_LINK=\"$CONTROL_DIR/rollback\""},{"line":37,"text":""},{"line":38,"text":"# shellcheck disable=SC1090"},{"line":39,"text":"source \"$ACTIVATION_FILE\""},{"line":40,"text":""},{"line":41,"text":"test \"${STAGE4I_ACTIVATION_STATUS:-}\" = \"ACTIVE\""},{"line":42,"text":"test -L \"$ACTIVE_LINK\""},{"line":43,"text":"test -L \"$ROLLBACK_LINK\""},{"line":44,"text":""},{"line":45,"text":"ACTIVE_RESOLVED=\"$(readlink -f \"$ACTIVE_LINK\")\""},{"line":46,"text":"ROLLBACK_RESOLVED=\"$(readlink -f \"$ROLLBACK_LINK\")\""},{"line":47,"text":""},{"line":48,"text":"test \"$ACTIVE_RESOLVED\" = \"${ACTIVE_PATH:-}\""},{"line":49,"text":"test \"$ROLLBACK_RESOLVED\" = \"${ROLLBACK_PATH:-}\""},{"line":50,"text":"test -d \"$ACTIVE_RESOLVED\""},{"line":51,"text":"test -d \"$ROLLBACK_RESOLVED\""},{"line":52,"text":""},{"line":53,"text":"META=\"$ACTIVE_RESOLVED/IMMUTABLE.env\""},{"line":54,"text":"MANIFEST=\"$ACTIVE_RESOLVED/MANIFEST.sha256\""},{"line":55,"text":""},{"line":56,"text":"test -f \"$META\""},{"line":57,"text":"test -f \"$MANIFEST\""},{"line":58,"text":"grep -q \"^BUILD_MARKER=$MARKER$\" \"$META\""},{"line":59,"text":"grep -q '^TASK_STAGE=4I$' \"$META\""},{"line":60,"text":"grep -q '^TASK_STATUS=IMMUTABLE$' \"$META\""},{"line":61,"text":""},{"line":62,"text":"("},{"line":63,"text":"    cd \"$ACTIVE_RESOLVED\""},{"line":64,"text":"    sha256sum -c MANIFEST.sha256 >/dev/null"},{"line":65,"text":")"},{"line":66,"text":""},{"line":67,"text":"if find \"$ACTIVE_RESOLVED\" \\"},{"line":68,"text":"    \\( ! -user root -o ! -group root \\) \\"},{"line":69,"text":"    -print -quit |"},{"line":70,"text":"    grep -q ."},{"line":71,"text":"then"},{"line":72,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=FAILED\""},{"line":73,"text":"    echo \"REASON=INVALID_OWNER\""},{"line":74,"text":"    exit 51"},{"line":75,"text":"fi"},{"line":76,"text":""},{"line":77,"text":"if find \"$ACTIVE_RESOLVED\" \\"},{"line":78,"text":"    -perm /0222 \\"},{"line":79,"text":"    -print -quit |"},{"line":80,"text":"    grep -q ."},{"line":81,"text":"then"},{"line":82,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=FAILED\""},{"line":83,"text":"    echo \"REASON=WRITABLE_IMMUTABLE_CONTENT\""},{"line":84,"text":"    exit 52"},{"line":85,"text":"fi"},{"line":86,"text":""},{"line":87,"text":"install -d -m 0750 -o root -g root \"$STATE_DIR\""},{"line":88,"text":""},{"line":89,"text":"{"},{"line":90,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=SUCCESS\""},{"line":91,"text":"    echo \"VERIFIED_AT_UTC=$(date -u +%Y-%m-%dT%H:%M:%SZ)\""},{"line":92,"text":"    echo \"ACTIVE_VERSION=${ACTIVE_VERSION:-UNKNOWN}\""},{"line":93,"text":"    echo \"ACTIVE_PATH=$ACTIVE_RESOLVED\""},{"line":94,"text":"    echo \"ROLLBACK_VERSION=${ROLLBACK_VERSION:-UNKNOWN}\""},{"line":95,"text":"    echo \"ROLLBACK_PATH=$ROLLBACK_RESOLVED\""},{"line":96,"text":"    echo \"MANIFEST_SHA256=$(sha256sum \"$MANIFEST\" | awk '{print $1}')\""},{"line":97,"text":"    echo \"VERIFIED_ON_HOST=$(hostname -f 2>/dev/null || hostname)\""},{"line":98,"text":"} >\"$TMP\""},{"line":99,"text":""},{"line":100,"text":"chown root:root \"$TMP\""},{"line":101,"text":"chmod 0440 \"$TMP\""},{"line":102,"text":"mv -f \"$TMP\" \"$REPORT\""},{"line":103,"text":"trap - EXIT"},{"line":104,"text":""},{"line":105,"text":"cat \"$REPORT\""}]},{"anchors":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":5,"text":"Requires=postgresql.service"},{"line":7,"text":"[Service]"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"binary":null,"bytes":616,"line_count":19,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","priority":90,"role":"unit-fragment","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":"Requires=postgresql.service"},{"line":6,"text":""},{"line":7,"text":"[Service]"},{"line":8,"text":"Type=oneshot"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":13,"text":"UMask=0027"},{"line":14,"text":"NoNewPrivileges=true"},{"line":15,"text":"PrivateTmp=true"},{"line":16,"text":"ProtectSystem=strict"},{"line":17,"text":"ProtectHome=true"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}]},{"anchors":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"binary":null,"bytes":231,"line_count":12,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","priority":90,"role":"unit-fragment","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=30s"},{"line":6,"text":"OnUnitActiveSec=60s"},{"line":7,"text":"AccuracySec=10s"},{"line":8,"text":"Persistent=true"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"},{"line":10,"text":""},{"line":11,"text":"[Install]"},{"line":12,"text":"WantedBy=timers.target"}]},{"anchors":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":3,"text":"After=network-online.target postgresql.service"},{"line":5,"text":"Requires=postgresql.service"},{"line":7,"text":"[Service]"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"binary":null,"bytes":803,"line_count":28,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","priority":90,"role":"unit-fragment","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":3,"text":"After=network-online.target postgresql.service"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":"Requires=postgresql.service"},{"line":6,"text":""},{"line":7,"text":"[Service]"},{"line":8,"text":"Type=simple"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":13,"text":"ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1"},{"line":14,"text":"Restart=on-failure"},{"line":15,"text":"RestartSec=5"},{"line":16,"text":"UMask=0027"},{"line":17,"text":"NoNewPrivileges=true"},{"line":18,"text":"PrivateTmp=true"},{"line":19,"text":"ProtectSystem=strict"},{"line":20,"text":"ProtectHome=true"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"},{"line":23,"text":"RestrictSUIDSGID=true"},{"line":24,"text":"LockPersonality=true"},{"line":25,"text":"RestrictRealtime=true"},{"line":26,"text":""},{"line":27,"text":"[Install]"},{"line":28,"text":"WantedBy=multi-user.target"}]},{"anchors":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":6,"text":"[Service]"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"binary":null,"bytes":1190,"line_count":42,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","priority":90,"role":"unit-fragment","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":3,"text":"After=network-online.target"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":""},{"line":6,"text":"[Service]"},{"line":7,"text":"Type=oneshot"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":12,"text":"UMask=0077"},{"line":13,"text":"NoNewPrivileges=yes"},{"line":14,"text":"PrivateTmp=yes"},{"line":15,"text":"PrivateDevices=yes"},{"line":16,"text":"ProtectSystem=strict"},{"line":17,"text":"ProtectHome=yes"},{"line":18,"text":"ProtectKernelTunables=yes"},{"line":19,"text":"ProtectKernelModules=yes"},{"line":20,"text":"ProtectKernelLogs=yes"},{"line":21,"text":"ProtectControlGroups=yes"},{"line":22,"text":"ProtectClock=yes"},{"line":23,"text":"ProtectHostname=yes"},{"line":24,"text":"LockPersonality=yes"},{"line":25,"text":"MemoryDenyWriteExecute=yes"},{"line":26,"text":"RestrictSUIDSGID=yes"},{"line":27,"text":"RestrictRealtime=yes"},{"line":28,"text":"RemoveIPC=yes"},{"line":29,"text":"CapabilityBoundingSet="},{"line":30,"text":"AmbientCapabilities="},{"line":31,"text":"RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6"},{"line":32,"text":"IPAddressDeny=any"},{"line":33,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":34,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":35,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":36,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":40,"text":"StateDirectoryMode=0750"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"},{"line":42,"text":"TimeoutStartSec=70"}]},{"anchors":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"binary":null,"bytes":223,"line_count":12,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","priority":90,"role":"unit-fragment","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=2min"},{"line":6,"text":"OnUnitActiveSec=60s"},{"line":7,"text":"RandomizedDelaySec=5s"},{"line":8,"text":"Persistent=true"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"},{"line":10,"text":""},{"line":11,"text":"[Install]"},{"line":12,"text":"WantedBy=timers.target"}]},{"anchors":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":4,"text":"[Service]"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"binary":null,"bytes":133,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","priority":90,"role":"unit-fragment","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}]},{"anchors":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"binary":null,"bytes":154,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","priority":90,"role":"unit-fragment","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run homelab cluster-admin full observer"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=5min"},{"line":6,"text":"OnUnitActiveSec=5min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]},{"anchors":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":4,"text":"[Service]"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"binary":null,"bytes":131,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","priority":90,"role":"unit-fragment","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}]},{"anchors":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"binary":null,"bytes":149,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","priority":90,"role":"unit-fragment","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run homelab cluster-admin observer"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=3min"},{"line":6,"text":"OnUnitActiveSec=5min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]},{"anchors":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":4,"text":"[Service]"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"binary":null,"bytes":127,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","priority":90,"role":"unit-fragment","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}]},{"anchors":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"binary":null,"bytes":151,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","priority":90,"role":"unit-fragment","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run homelab cluster-admin self check"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=2min"},{"line":6,"text":"OnUnitActiveSec=5min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]}],"transaction_gate":{"frontend_index_exact":true,"ready_for_final_atomic_design":true,"reference_backend_exact":true,"remote_root_probe":true,"selected_source_count":18,"unit_count":16},"working_directories":["/opt/cluster-admin-incident-engine"]},"rollback_restored":false,"rollback_started":false,"schema":19,"status":"OK","v18_exact_result":{"capsule_sha256":"59b7a04f38f9ccf08b499fea9082c9864bebf360c6f33d25502e3811ab762db6","changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z","control_plane_error":{"message":"existing runner returned a different rc after writing its public artifact; runtime publication or wrapper completion failed","type":"RunnerWrapperMismatch"},"declared_report_bytes":193466,"declared_report_sha256":"772f2c3f77d7909de34aa159e0e2c8c2ca5480ff51cbcd4bcb194af9019b43d0","identity_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":194314,"output_sha256":"5b17a087bd51a9f4e901cd820e129160e70081636d2da63d902f481305df4f99","output_source":"runner_json","rollback_restored":null,"rollback_started":false,"runner_document_rc":0,"runner_json_sha256":"a73b7ed1a9bca8aac9fe10824f3921f6a45e1898091a9f8caafdd3dc699f28d7","runner_rc":75,"runner_status":"OK","runner_text_sha256":"924eb43f0b1633f8b40a49cae8d04af57f9fbd82e61acc8d083c2e3050d7ba6f","runner_wrapper_matches_document":false,"sanitized_report_bytes":193386,"sanitized_report_sha256":"2a7d38c6c1a684e26469ec611090b0916fbefb54f1621f84dbe3ae6672c599bb","state_document_sha256":"212ada5f20b26b3a7b1044ef597d7562c83666f391f1b9cbb0d91bf899e9a95d","state_path_sha256":"cc9ddb1c213dedb6c87405dd161acb5b8cd92b28cfdf6a153a03e7728ce9e456","state_status":"FAIL"}}

OUTPUT_END
CHAT_OUTPUT_END