homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity
Files
d4dec2f370d04a7e2d3b7065b8c11da8982a59cd
homelab-public-context/runtime/history/HOMELAB-CLUSTER-ADMIN-UPDATES-V19-REMOTE-CORE-READ-ONLY-20260805T133000Z.json
T

39 lines
87 KiB
JSON
Raw Blame History

{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V19-REMOTE-CORE-READ-ONLY-20260805T133000Z",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "verify",
"component": "homelab-cluster-admin-update-visibility",
"started_at_utc": "2026-08-05T13:44:09Z",
"finished_at_utc": "2026-08-05T13:44:10Z",
"reference_register_checked": true,
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
"error_register_checked": true,
"error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
"command_sha256": "ed901eae70df3049b86d3644e4a1166315c959eff5c2a64715b75ce544b7abf4",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
"change_declared": false,
"result_contract_valid": true,
"result_contract_status": null,
"result_contract_error": null,
"command_rc": 0,
"changes_made": false,
"rollback_started": false,
"rollback_restored": null,
"mutation_outcome": "NO_MUTATION",
"sanitized": true,
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "136221f755ac0a721469151448174aef1afce2fb7597afcd8ba4652284099fc9",
"sanitized_output_sha256": "136221f755ac0a721469151448174aef1afce2fb7597afcd8ba4652284099fc9",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V19-REMOTE-CORE-READ-ONLY-20260805T133000Z\",\"command_rc\":0,\"control_plane_error\":null,\"frontend_index_count\":1,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_final_atomic_design\":true,\"reference_backend_count\":1,\"relevant_unit_count\":16,\"remote_probe_root\":true,\"report_bytes\":78679,\"report_sha256\":\"82e69b5c37c04ad98c95899686cd170fa289007b289de00cede181014256ec87\",\"rollback_restored\":false,\"rollback_started\":false,\"selected_source_count\":18,\"status\":\"OK\",\"v18_exact_result_identity_confirmed\":true,\"v18_output_sha256\":\"5b17a087bd51a9f4e901cd820e129160e70081636d2da63d902f481305df4f99\",\"v18_runner_wrapper_matches_document\":false,\"version\":1}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V19-REMOTE-CORE-READ-ONLY-20260805T133000Z\",\"mutation_outcome\":\"NO_MUTATION\",\"next_gate\":{\"next_required_action\":\"build one corrected atomic local+remote transaction\",\"ready_for_final_atomic_design\":true},\"read_only\":true,\"remote_core\":{\"exact_preconditions\":{\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\":{\"bytes\":616,\"mode\":\"0o644\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\":{\"bytes\":231,\"mode\":\"0o644\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},\"/etc/systemd/system/cluster-admin-incident-engine.service\":{\"bytes\":803,\"mode\":\"0o644\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},\"/etc/systemd/system/cluster-admin-probe-agent.service\":{\"bytes\":1190,\"mode\":\"0o644\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},\"/etc/systemd/system/cluster-admin-probe-agent.timer\":{\"bytes\":223,\"mode\":\"0o644\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\":{\"bytes\":133,\"mode\":\"0o644\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\":{\"bytes\":154,\"mode\":\"0o644\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},\"/etc/systemd/system/homelab-cluster-admin-observer.service\":{\"bytes\":131,\"mode\":\"0o644\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},\"/etc/systemd/system/homelab-cluster-admin-observer.timer\":{\"bytes\":149,\"mode\":\"0o644\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},\"/etc/systemd/system/homelab-cluster-admin-self-check.service\":{\"bytes\":127,\"mode\":\"0o644\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\":{\"bytes\":151,\"mode\":\"0o644\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\":{\"bytes\":138,\"mode\":\"0o644\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\":{\"bytes\":154,\"mode\":\"0o644\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\":{\"bytes\":301,\"mode\":\"0o644\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},\"/etc/systemd/system/homelab-stage4i-verify.service\":{\"bytes\":465,\"mode\":\"0o644\",\"sha256\":\"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b\"},\"/etc/systemd/system/homelab-stage4i-verify.timer\":{\"bytes\":224,\"mode\":\"0o644\",\"sha256\":\"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396\"},\"/opt/homelab-cluster-admin/cluster-observer.sh\":{\"bytes\":2832,\"mode\":\"0o755\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},\"/opt/homelab-cluster-admin/full-observer.sh\":{\"bytes\":3471,\"mode\":\"0o755\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\":{\"bytes\":1433,\"mode\":\"0o750\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},\"/opt/homelab-cluster-admin/render-webpanel.sh\":{\"bytes\":3708,\"mode\":\"0o755\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},\"/opt/homelab-cluster-admin/self-check.sh\":{\"bytes\":848,\"mode\":\"0o755\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},\"/usr/local/sbin/homelab-stage4i-verify\":{\"bytes\":2541,\"mode\":\"0o750\",\"sha256\":\"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d\"},\"/var/www/homelab-cluster-admin/index.html\":{\"bytes\":10353,\"mode\":\"0o644\",\"sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\"}},\"exec_paths\":[\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"/opt/homelab-cluster-admin/full-observer.sh\",\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"/opt/homelab-cluster-admin/self-check.sh\",\"/usr/bin/python3\",\"/usr/local/sbin/homelab-stage4i-verify\"],\"fragment_paths\":[\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"/etc/systemd/system/homelab-stage4i-verify.service\",\"/etc/systemd/system/homelab-stage4i-verify.timer\"],\"gate\":{\"exact_source_count\":30,\"index_source_count\":1,\"ready_for_atomic_design\":true,\"reference_source_count\":1,\"unit_record_count\":16},\"index_path\":\"/var/www/homelab-cluster-admin/index.html\",\"reference_path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"relevant_unit_count\":16,\"relevant_units\":[{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006178 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],<PRIVATE_IPV4>\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:27 UTC] ; stop_time=[Wed 2026-08-05 13:20:28 UTC] ; pid=2006164 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:15 UTC] ; pid=2005776 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005777 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005778 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006179 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:11:27 UTC] ; stop_time=[Wed 2026-08-05 13:11:27 UTC] ; pid=2004608 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"Group\":\"root\",\"LoadState\":\"loaded\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"root\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}],\"remote_euid\":0,\"remote_hostname\":\"cluster-admin\",\"selected_source_count\":18,\"selected_sources\":[{\"anchors\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":39,\"text\":\"cat > \\\"$WEB/index.html\\\" <<HTML\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"binary\":null,\"bytes\":3708,\"line_count\":79,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"priority\":100,\"role\":\"reference-backend\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":5,\"text\":\"mkdir -p \\\"$WEB\\\"\"},{\"line\":6,\"text\":\"BAD=0\"},{\"line\":7,\"text\":\"\"},{\"line\":8,\"text\":\"line_or_missing() {\"},{\"line\":9,\"text\":\" f=\\\"$1\\\"\"},{\"line\":10,\"text\":\" if [ -s \\\"$f\\\" ]; then head -n1 \\\"$f\\\"; else echo \\\"STATUS=WARN TYPE=missing FILE=$f\\\"; fi\"},{\"line\":11,\"text\":\"}\"},{\"line\":12,\"text\":\"\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":20,\"text\":\"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":21,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":22,\"text\":\"\"},{\"line\":23,\"text\":\"for line in \\\"$SELF\\\" \\\"$OBSERVER\\\" \\\"$FULL\\\" \\\"$BACKUP\\\"; do\"},{\"line\":24,\"text\":\" echo \\\"$line\\\" | grep -q '^STATUS=OK' || BAD=$((BAD+1))\"},{\"line\":25,\"text\":\"done\"},{\"line\":26,\"text\":\"echo \\\"$FULL\\\" | grep -q 'BAD=0' || BAD=$((BAD+1))\"},{\"line\":27,\"text\":\"echo \\\"$FULL\\\" | grep -q 'WARN=0' || BAD=$((BAD+1))\"},{\"line\":28,\"text\":\"echo \\\"$BACKUP\\\" | grep -q 'BAD=0' || BAD=$((BAD+1))\"},{\"line\":29,\"text\":\"echo \\\"$BACKUP\\\" | grep -q 'MAIL_CLOUD_2COPY_VERIFY_OK' || BAD=$((BAD+1))\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":31,\"text\":\"[ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":32,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":33,\"text\":\"\"},{\"line\":34,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; COLOR=\\\"#0a7f38\\\"; BADGE=\\\"OK\\\"; else STATUS=WARN; COLOR=\\\"#b26a00\\\"; BADGE=\\\"REVIEW\\\"; fi\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":37,\"text\":\"echo \\\"$STATUS_LINE\\\" > \\\"$WEB/status.txt\\\"\"},{\"line\":38,\"text\":\"\"},{\"line\":39,\"text\":\"cat > \\\"$WEB/index.html\\\" <<HTML\"},{\"line\":40,\"text\":\"<!doctype html>\"},{\"line\":41,\"text\":\"<html lang=\\\"ru\\\">\"},{\"line\":42,\"text\":\"<head>\"},{\"line\":43,\"text\":\"<meta charset=\\\"utf-8\\\">\"},{\"line\":44,\"text\":\"<meta http-equiv=\\\"refresh\\\" content=\\\"60\\\">\"},{\"line\":45,\"text\":\"<meta name=\\\"viewport\\\" content=\\\"width=device-width, initial-scale=1\\\">\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":47,\"text\":\"<style>\"},{\"line\":48,\"text\":\"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}\"},{\"line\":49,\"text\":\".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}\"},{\"line\":50,\"text\":\"h1{margin:0 0 8px}\"},{\"line\":51,\"text\":\".badge{display:inline-block;background:$COLOR;color:white;padding:8px 14px;border-radius:999px;font-weight:700}\"},{\"line\":52,\"text\":\"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}\"},{\"line\":53,\"text\":\".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}\"},{\"line\":54,\"text\":\".small{color:#aab2c0}\"},{\"line\":55,\"text\":\"</style>\"},{\"line\":56,\"text\":\"</head>\"},{\"line\":57,\"text\":\"<body>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":59,\"text\":\"<div class=\\\"small\\\">generated UTC: $TS · VM180 · [PRIVATE_IP] · refresh 60s</div>\"},{\"line\":60,\"text\":\"<p><span class=\\\"badge\\\">$BADGE</span></p>\"},{\"line\":61,\"text\":\"\"},{\"line\":62,\"text\":\"<div class=\\\"grid\\\">\"},{\"line\":63,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>$SELF\"},{\"line\":64,\"text\":\"FAILED_UNITS=$FAILED\"},{\"line\":65,\"text\":\"QGA=$QGA</pre></div>\"},{\"line\":66,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>$OBSERVER</pre></div>\"},{\"line\":67,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>$FULL</pre></div>\"},{\"line\":68,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>$BACKUP</pre></div>\"},{\"line\":69,\"text\":\"</div>\"},{\"line\":70,\"text\":\"\"},{\"line\":71,\"text\":\"<div class=\\\"card\\\"><h2>Overall</h2><pre>$OVERALL</pre></div>\"},{\"line\":72,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>$FINAL</pre></div>\"},{\"line\":73,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>$STRICT</pre></div>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":75,\"text\":\"</body>\"},{\"line\":76,\"text\":\"</html>\"},{\"line\":77,\"text\":\"HTML\"},{\"line\":78,\"text\":\"\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}]},{\"anchors\":[{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":45,\"text\":\"## Service map checks\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"}],\"binary\":null,\"bytes\":10353,\"line_count\":80,\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"priority\":95,\"role\":\"frontend-index\",\"sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"source\":[{\"line\":1,\"text\":\"<!doctype html>\"},{\"line\":2,\"text\":\"<html lang=\\\"ru\\\">\"},{\"line\":3,\"text\":\"<head>\"},{\"line\":4,\"text\":\"<meta charset=\\\"utf-8\\\">\"},{\"line\":5,\"text\":\"<meta http-equiv=\\\"refresh\\\" content=\\\"60\\\">\"},{\"line\":6,\"text\":\"<meta name=\\\"viewport\\\" content=\\\"width=device-width, initial-scale=1\\\">\"},{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":8,\"text\":\"<style>\"},{\"line\":9,\"text\":\"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}\"},{\"line\":10,\"text\":\".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}\"},{\"line\":11,\"text\":\"h1{margin:0 0 8px}\"},{\"line\":12,\"text\":\".badge{display:inline-block;background:#b26a00;color:white;padding:8px 14px;border-radius:999px;font-weight:700}\"},{\"line\":13,\"text\":\"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}\"},{\"line\":14,\"text\":\".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}\"},{\"line\":15,\"text\":\".small{color:#aab2c0}\"},{\"line\":16,\"text\":\"</style>\"},{\"line\":17,\"text\":\"</head>\"},{\"line\":18,\"text\":\"<body>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":20,\"text\":\"<div class=\\\"small\\\">generated UTC: 2026-08-05T13:20:48Z · VM180 · [PRIVATE_IP] · refresh 60s</div>\"},{\"line\":21,\"text\":\"<p><span class=\\\"badge\\\">REVIEW</span></p>\"},{\"line\":22,\"text\":\"\"},{\"line\":23,\"text\":\"<div class=\\\"grid\\\">\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":25,\"text\":\"FAILED_UNITS=1\"},{\"line\":26,\"text\":\"QGA=active</pre></div>\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":30,\"text\":\"</div>\"},{\"line\":31,\"text\":\"\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":36,\"text\":\"generated_utc=2026-08-05T13:19:08Z\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":38,\"text\":\"\"},{\"line\":39,\"text\":\"## Restricted probes\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":44,\"text\":\"\"},{\"line\":45,\"text\":\"## Service map checks\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":47,\"text\":\"OK severity=critical kind=tcp name=pve01_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":48,\"text\":\"OK severity=critical kind=tcp name=pve01_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\"},{\"line\":49,\"text\":\"OK severity=critical kind=tcp name=pve01_health target=[PRIVATE_IP] value=9101 tcp_9101_rc=0\"},{\"line\":50,\"text\":\"OK severity=critical kind=tcp name=pve02_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":51,\"text\":\"OK severity=critical kind=tcp name=pve02_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\"},{\"line\":52,\"text\":\"OK severity=critical kind=tcp name=pve03_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":53,\"text\":\"OK severity=critical kind=tcp name=pve03_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\"},{\"line\":54,\"text\":\"OK severity=critical kind=tcp name=dns1_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0\"},{\"line\":55,\"text\":\"OK severity=critical kind=tcp name=dns2_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0\"},{\"line\":56,\"text\":\"OK severity=critical kind=tcp name=edge_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":58,\"text\":\"OK severity=critical kind=ping name=vm130_edge_vm_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":59,\"text\":\"OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":60,\"text\":\"OK severity=critical kind=ping name=vm150_nextcloud_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":61,\"text\":\"OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":62,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\"},{\"line\":63,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":64,\"text\":\"OK severity=critical kind=ping name=vm160_forum_prod_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":65,\"text\":\"OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":66,\"text\":\"OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=0\"},{\"line\":67,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":68,\"text\":\"OK severity=critical kind=ping name=vm170_core_apps_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":69,\"text\":\"OK severity=critical kind=tcp name=vm170_core_apps_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":70,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\"},{\"line\":71,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":72,\"text\":\"OK severity=critical kind=ping name=vm171_monitoring_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":73,\"text\":\"OK severity=critical kind=tcp name=vm171_monitoring_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":74,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\"},{\"line\":75,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":76,\"text\":\"\"},{\"line\":77,\"text\":\"## Result\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":79,\"text\":\"</body>\"},{\"line\":80,\"text\":\"</html>\"}]},{\"anchors\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\" line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\" echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"binary\":null,\"bytes\":2832,\"line_count\":96,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"priority\":92,\"role\":\"exec-target\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":4,\"text\":\"exec 9>\\\"$LOCK\\\"\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":6,\"text\":\"\"},{\"line\":7,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":10,\"text\":\"REPORT=\\\"$ROOT/latest.md\\\"\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":12,\"text\":\"TMP=\\\"$REPORT.tmp\\\"\"},{\"line\":13,\"text\":\"mkdir -p \\\"$ROOT\\\" \\\"$HEALTH_DIR\\\"\"},{\"line\":14,\"text\":\"BAD=0\"},{\"line\":15,\"text\":\"WARN=0\"},{\"line\":16,\"text\":\"\"},{\"line\":17,\"text\":\"check_ping() {\"},{\"line\":18,\"text\":\" name=\\\"$1\\\"; ip=\\\"$2\\\"\"},{\"line\":19,\"text\":\" ping -c1 -W1 \\\"$ip\\\" >/dev/null 2>&1\"},{\"line\":20,\"text\":\" rc=$?\"},{\"line\":21,\"text\":\" echo \\\"$name ping rc=$rc\\\" >> \\\"$TMP\\\"\"},{\"line\":22,\"text\":\" [ \\\"$rc\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":23,\"text\":\"}\"},{\"line\":24,\"text\":\"\"},{\"line\":25,\"text\":\"check_tcp() {\"},{\"line\":26,\"text\":\" name=\\\"$1\\\"; ip=\\\"$2\\\"; port=\\\"$3\\\"\"},{\"line\":27,\"text\":\" timeout 2 bash -c \\\"</dev/tcp/$ip/$port\\\" >/dev/null 2>&1\"},{\"line\":28,\"text\":\" rc=$?\"},{\"line\":29,\"text\":\" echo \\\"$name tcp/$port rc=$rc\\\" >> \\\"$TMP\\\"\"},{\"line\":30,\"text\":\" [ \\\"$rc\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":31,\"text\":\"}\"},{\"line\":32,\"text\":\"\"},{\"line\":33,\"text\":\"check_http_status_ok() {\"},{\"line\":34,\"text\":\" name=\\\"$1\\\"; url=\\\"$2\\\"\"},{\"line\":35,\"text\":\" line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":36,\"text\":\" rc=$?\"},{\"line\":37,\"text\":\" echo \\\"$name http rc=$rc line=$line\\\" >> \\\"$TMP\\\"\"},{\"line\":38,\"text\":\" echo \\\"$line\\\" | grep -q \\\"^STATUS=\\\"\"},{\"line\":39,\"text\":\" okrc=$?\"},{\"line\":40,\"text\":\" [ \\\"$rc\\\" = 0 ] && [ \\\"$okrc\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":41,\"text\":\"}\"},{\"line\":42,\"text\":\"\"},{\"line\":43,\"text\":\"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":44,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":46,\"text\":\"\"},{\"line\":47,\"text\":\"{\"},{\"line\":48,\"text\":\" echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":49,\"text\":\" echo \\\"generated_utc=$TS\\\"\"},{\"line\":50,\"text\":\" echo\"},{\"line\":51,\"text\":\" echo \\\"## Self\\\"\"},{\"line\":52,\"text\":\" echo \\\"host=$(hostname 2>/dev/null)\\\"\"},{\"line\":53,\"text\":\" echo \\\"failed_units=$FAILED\\\"\"},{\"line\":54,\"text\":\" echo \\\"qga=$QGA\\\"\"},{\"line\":55,\"text\":\" echo \\\"self_check=$SELF_LINE\\\"\"},{\"line\":56,\"text\":\" echo\"},{\"line\":57,\"text\":\" echo \\\"## Network checks\\\"\"},{\"line\":58,\"text\":\"} > \\\"$TMP\\\"\"},{\"line\":59,\"text\":\"\"},{\"line\":60,\"text\":\"[ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":61,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":62,\"text\":\"echo \\\"$SELF_LINE\\\" | grep -q \\\"^STATUS=OK\\\" || BAD=$((BAD+1))\"},{\"line\":63,\"text\":\"\"},{\"line\":64,\"text\":\"check_ping pve01 [PRIVATE_IP]\"},{\"line\":65,\"text\":\"check_tcp pve01_ssh [PRIVATE_IP] 22\"},{\"line\":66,\"text\":\"check_tcp pve01_proxmox [PRIVATE_IP] 8006\"},{\"line\":67,\"text\":\"check_tcp pve01_health [PRIVATE_IP] 9101\"},{\"line\":68,\"text\":\"\"},{\"line\":69,\"text\":\"check_ping pve02 [PRIVATE_IP]\"},{\"line\":70,\"text\":\"check_tcp pve02_ssh [PRIVATE_IP] 22\"},{\"line\":71,\"text\":\"check_tcp pve02_proxmox [PRIVATE_IP] 8006\"},{\"line\":72,\"text\":\"\"},{\"line\":73,\"text\":\"check_ping pve03 [PRIVATE_IP]\"},{\"line\":74,\"text\":\"check_tcp pve03_ssh [PRIVATE_IP] 22\"},{\"line\":75,\"text\":\"check_tcp pve03_proxmox [PRIVATE_IP] 8006\"},{\"line\":76,\"text\":\"\"},{\"line\":77,\"text\":\"check_ping edge [PRIVATE_IP]\"},{\"line\":78,\"text\":\"check_tcp edge_ssh [PRIVATE_IP] 22\"},{\"line\":79,\"text\":\"\"},{\"line\":80,\"text\":\"check_ping dns1 [PRIVATE_IP]\"},{\"line\":81,\"text\":\"check_tcp dns1_tcp53 [PRIVATE_IP] 53\"},{\"line\":82,\"text\":\"check_ping dns2 [PRIVATE_IP]\"},{\"line\":83,\"text\":\"check_tcp dns2_tcp53 [PRIVATE_IP] 53\"},{\"line\":84,\"text\":\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":86,\"text\":\"\"},{\"line\":87,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"},{\"line\":89,\"text\":\"echo \\\"$LINE\\\" | tee \\\"$HEALTH\\\" >/dev/null\"},{\"line\":90,\"text\":\"{\"},{\"line\":91,\"text\":\" echo\"},{\"line\":92,\"text\":\" echo \\\"## Result\\\"\"},{\"line\":93,\"text\":\" echo \\\"$LINE\\\"\"},{\"line\":94,\"text\":\"} >> \\\"$TMP\\\"\"},{\"line\":95,\"text\":\"mv \\\"$TMP\\\" \\\"$REPORT\\\"\"},{\"line\":96,\"text\":\"cat \\\"$HEALTH\\\"\"}]},{\"anchors\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\" echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\" echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":46,\"text\":\" echo \\\"## Service map checks\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"binary\":null,\"bytes\":3471,\"line_count\":113,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"priority\":92,\"role\":\"exec-target\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":7,\"text\":\"REPORT=\\\"$ROOT/full-observer-latest.md\\\"\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":10,\"text\":\"mkdir -p \\\"$ROOT\\\" \\\"$HEALTH_DIR\\\"\"},{\"line\":11,\"text\":\"exec 9>\\\"$LOCK\\\"\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":13,\"text\":\"\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":15,\"text\":\"BAD=0\"},{\"line\":16,\"text\":\"WARN=0\"},{\"line\":17,\"text\":\"INFO_OFF=0\"},{\"line\":18,\"text\":\"TOTAL=0\"},{\"line\":19,\"text\":\"CRIT_FAIL=0\"},{\"line\":20,\"text\":\"OPT_OFF=0\"},{\"line\":21,\"text\":\"PROBE_OK=0\"},{\"line\":22,\"text\":\"PROBE_BAD=0\"},{\"line\":23,\"text\":\"\"},{\"line\":24,\"text\":\"TMP=\\\"$REPORT.tmp\\\"\"},{\"line\":25,\"text\":\"{\"},{\"line\":26,\"text\":\" echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":27,\"text\":\" echo \\\"generated_utc=$TS\\\"\"},{\"line\":28,\"text\":\" echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":29,\"text\":\" echo\"},{\"line\":30,\"text\":\" echo \\\"## Restricted probes\\\"\"},{\"line\":31,\"text\":\"} > \\\"$TMP\\\"\"},{\"line\":32,\"text\":\"\"},{\"line\":33,\"text\":\"for t in \\\"root@[PRIVATE_IP] pve01\\\" \\\"root@[PRIVATE_IP] pve02\\\" \\\"root@[PRIVATE_IP] pve03\\\" \\\"debian@[PRIVATE_IP] edge\\\"; do\"},{\"line\":34,\"text\":\" set -- $t\"},{\"line\":35,\"text\":\" target=\\\"$1\\\"; name=\\\"$2\\\"\"},{\"line\":36,\"text\":\" out=$(ssh -n -i \\\"$K\\\" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 \\\"$target\\\" probe 2>/tmp/full-probe-$name.err)\"},{\"line\":37,\"text\":\" rc=$?\"},{\"line\":38,\"text\":\" echo \\\"$name ssh_rc=$rc $out\\\" >> \\\"$TMP\\\"\"},{\"line\":39,\"text\":\" echo \\\"$out\\\" | grep -q '^STATUS=OK'\"},{\"line\":40,\"text\":\" ok=$?\"},{\"line\":41,\"text\":\" if [ \\\"$rc\\\" = 0 ] && [ \\\"$ok\\\" = 0 ]; then PROBE_OK=$((PROBE_OK+1)); else PROBE_BAD=$((PROBE_BAD+1)); BAD=$((BAD+1)); fi\"},{\"line\":42,\"text\":\"done\"},{\"line\":43,\"text\":\"\"},{\"line\":44,\"text\":\"{\"},{\"line\":45,\"text\":\" echo\"},{\"line\":46,\"text\":\" echo \\\"## Service map checks\\\"\"},{\"line\":47,\"text\":\"} >> \\\"$TMP\\\"\"},{\"line\":48,\"text\":\"\"},{\"line\":49,\"text\":\"while IFS=\\\"$(printf '\\\\t')\\\" read -r severity kind name target value; do\"},{\"line\":50,\"text\":\" [ -n \\\"$severity\\\" ] || continue\"},{\"line\":51,\"text\":\" TOTAL=$((TOTAL+1))\"},{\"line\":52,\"text\":\" rc=0\"},{\"line\":53,\"text\":\" detail=\\\"\\\"\"},{\"line\":54,\"text\":\" case \\\"$kind\\\" in\"},{\"line\":55,\"text\":\" ping)\"},{\"line\":56,\"text\":\" ping -c1 -W1 \\\"$target\\\" >/dev/null 2>&1\"},{\"line\":57,\"text\":\" rc=$?\"},{\"line\":58,\"text\":\" detail=\\\"ping_rc=$rc\\\"\"},{\"line\":59,\"text\":\" ;;\"},{\"line\":60,\"text\":\" tcp)\"},{\"line\":61,\"text\":\" timeout 2 bash -c \\\"</dev/tcp/$target/$value\\\" >/dev/null 2>&1\"},{\"line\":62,\"text\":\" rc=$?\"},{\"line\":63,\"text\":\" detail=\\\"tcp_${value}_rc=$rc\\\"\"},{\"line\":64,\"text\":\" ;;\"},{\"line\":65,\"text\":\" http_status_ok)\"},{\"line\":66,\"text\":\" line=$(curl -fsS --max-time 5 \\\"$value\\\" 2>/tmp/full-http-$name.err | head -n1)\"},{\"line\":67,\"text\":\" curlrc=$?\"},{\"line\":68,\"text\":\" echo \\\"$line\\\" | grep -q '^STATUS='\"},{\"line\":69,\"text\":\" okrc=$?\"},{\"line\":70,\"text\":\" [ \\\"$curlrc\\\" = 0 ] && [ \\\"$okrc\\\" = 0 ]\"},{\"line\":71,\"text\":\" rc=$?\"},{\"line\":72,\"text\":\" detail=\\\"http_rc=$curlrc ok_rc=$okrc line=$line\\\"\"},{\"line\":73,\"text\":\" ;;\"},{\"line\":74,\"text\":\" missing)\"},{\"line\":75,\"text\":\" rc=1\"},{\"line\":76,\"text\":\" detail=\\\"missing_target=$target value=$value\\\"\"},{\"line\":77,\"text\":\" ;;\"},{\"line\":78,\"text\":\" *)\"},{\"line\":79,\"text\":\" rc=1\"},{\"line\":80,\"text\":\" detail=\\\"unknown_kind=$kind\\\"\"},{\"line\":81,\"text\":\" ;;\"},{\"line\":82,\"text\":\" esac\"},{\"line\":83,\"text\":\"\"},{\"line\":84,\"text\":\" if [ \\\"$rc\\\" = 0 ]; then\"},{\"line\":85,\"text\":\" echo \\\"OK severity=$severity kind=$kind name=$name target=$target value=$value $detail\\\" >> \\\"$TMP\\\"\"},{\"line\":86,\"text\":\" else\"},{\"line\":87,\"text\":\" if [ \\\"$severity\\\" = critical ]; then\"},{\"line\":88,\"text\":\" echo \\\"FAIL severity=critical kind=$kind name=$name target=$target value=$value $detail\\\" >> \\\"$TMP\\\"\"},{\"line\":89,\"text\":\" CRIT_FAIL=$((CRIT_FAIL+1))\"},{\"line\":90,\"text\":\" BAD=$((BAD+1))\"},{\"line\":91,\"text\":\" else\"},{\"line\":92,\"text\":\" echo \\\"INFO_OFF severity=optional kind=$kind name=$name target=$target value=$value $detail\\\" >> \\\"$TMP\\\"\"},{\"line\":93,\"text\":\" OPT_OFF=$((OPT_OFF+1))\"},{\"line\":94,\"text\":\" INFO_OFF=$((INFO_OFF+1))\"},{\"line\":95,\"text\":\" fi\"},{\"line\":96,\"text\":\" fi\"},{\"line\":97,\"text\":\"done < \\\"$MAP\\\"\"},{\"line\":98,\"text\":\"\"},{\"line\":99,\"text\":\"SELF_FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":100,\"text\":\"[ \\\"$SELF_FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":101,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":102,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":103,\"text\":\"\"},{\"line\":104,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"},{\"line\":106,\"text\":\"echo \\\"$LINE\\\" | tee \\\"$HEALTH\\\" >/dev/null\"},{\"line\":107,\"text\":\"{\"},{\"line\":108,\"text\":\" echo\"},{\"line\":109,\"text\":\" echo \\\"## Result\\\"\"},{\"line\":110,\"text\":\" echo \\\"$LINE\\\"\"},{\"line\":111,\"text\":\"} >> \\\"$TMP\\\"\"},{\"line\":112,\"text\":\"mv \\\"$TMP\\\" \\\"$REPORT\\\"\"},{\"line\":113,\"text\":\"cat \\\"$HEALTH\\\"\"}]},{\"anchors\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"binary\":null,\"bytes\":1433,\"line_count\":27,\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"priority\":92,\"role\":\"exec-target\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set -Eeuo pipefail\"},{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"},{\"line\":6,\"text\":\"ORIGINAL_RC=0\"},{\"line\":7,\"text\":\"\\\"$ORIGINAL\\\" || ORIGINAL_RC=$?\"},{\"line\":8,\"text\":\"[ -s \\\"$HEALTH\\\" ] || exit \\\"$ORIGINAL_RC\\\"\"},{\"line\":9,\"text\":\"line=\\\"$(sed -n \\\"1p\\\" \\\"$HEALTH\\\" 2>/dev/null || true)\\\"\"},{\"line\":10,\"text\":\"field(){ local key=<REDACTED>\"},{\"line\":11,\"text\":\"self_failed=\\\"$(field SELF_FAILED_UNITS || true)\\\"\"},{\"line\":12,\"text\":\"qga_state=\\\"$(field QGA || true)\\\"\"},{\"line\":13,\"text\":\"backup_state=\\\"$(field BACKUP || true)\\\"\"},{\"line\":14,\"text\":\"if [ \\\"$ORIGINAL_RC\\\" -eq 0 ] && [ \\\"$self_failed\\\" = \\\"0\\\" ] && [ \\\"$qga_state\\\" = \\\"active\\\" ] && [ \\\"$backup_state\\\" = \\\"MAIL_CLOUD_2COPY_VERIFY_OK\\\" ]; then\"},{\"line\":15,\"text\":\" normalized=\\\"$(printf \\\"%s\\\\n\\\" \\\"$line\\\" | sed -E \\\"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\\\")\\\"\"},{\"line\":16,\"text\":\" printf \\\"%s\\\\n\\\" \\\"$normalized\\\" > \\\"$HEALTH\\\"\"},{\"line\":17,\"text\":\" if [ -f \\\"$STATUS_FILE\\\" ]; then\"},{\"line\":18,\"text\":\" status_line=\\\"$(sed -n \\\"1p\\\" \\\"$STATUS_FILE\\\" 2>/dev/null || true)\\\"\"},{\"line\":19,\"text\":\" status_normalized=\\\"$(printf \\\"%s\\\\n\\\" \\\"$status_line\\\" | sed -E \\\"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\\\")\\\"\"},{\"line\":20,\"text\":\" tmp=\\\"${STATUS_FILE}.tmp.$$\\\"\"},{\"line\":21,\"text\":\" { printf \\\"%s\\\\n\\\" \\\"$status_normalized\\\"; tail -n +2 \\\"$STATUS_FILE\\\" 2>/dev/null || true; } > \\\"$tmp\\\"\"},{\"line\":22,\"text\":\" chown --reference=\\\"$STATUS_FILE\\\" \\\"$tmp\\\" 2>/dev/null || true\"},{\"line\":23,\"text\":\" chmod --reference=\\\"$STATUS_FILE\\\" \\\"$tmp\\\" 2>/dev/null || true\"},{\"line\":24,\"text\":\" mv \\\"$tmp\\\" \\\"$STATUS_FILE\\\"\"},{\"line\":25,\"text\":\" fi\"},{\"line\":26,\"text\":\"fi\"},{\"line\":27,\"text\":\"exit \\\"$ORIGINAL_RC\\\"\"}]},{\"anchors\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"binary\":null,\"bytes\":848,\"line_count\":17,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"priority\":92,\"role\":\"exec-target\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"BAD=0\"},{\"line\":5,\"text\":\"HOST=$(hostname 2>/dev/null || echo unknown)\"},{\"line\":6,\"text\":\"IP_OK=0\"},{\"line\":7,\"text\":\"ip -4 -br a | grep -q \\\"[PRIVATE_IP]\\\" && IP_OK=1 || BAD=$((BAD+1))\"},{\"line\":8,\"text\":\"DNS_OK=0\"},{\"line\":9,\"text\":\"getent hosts pve01 >/dev/null 2>&1 || getent hosts gram1.ru >/dev/null 2>&1\"},{\"line\":10,\"text\":\"[ \\\"$?\\\" = 0 ] && DNS_OK=1 || BAD=$((BAD+1))\"},{\"line\":11,\"text\":\"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":12,\"text\":\"[ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":13,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":14,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":15,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}]},{\"anchors\":[{\"line\":6,\"text\":\"STATE_DIR=\\\"/var/lib/homelab-cluster-admin/stage4i-verification\\\"\"},{\"line\":17,\"text\":\" /var/lib/homelab-cluster-admin \\\\\"}],\"binary\":null,\"bytes\":2541,\"line_count\":105,\"mode\":\"0o750\",\"path\":\"/usr/local/sbin/homelab-stage4i-verify\",\"priority\":92,\"role\":\"exec-target\",\"sha256\":\"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set -euo pipefail\"},{\"line\":3,\"text\":\"export LC_ALL=C\"},{\"line\":4,\"text\":\"\"},{\"line\":5,\"text\":\"MARKER=\\\"CR0019_STAGE4I_IMMUTABLE_V1\\\"\"},{\"line\":6,\"text\":\"STATE_DIR=\\\"/var/lib/homelab-cluster-admin/stage4i-verification\\\"\"},{\"line\":7,\"text\":\"REPORT=\\\"$STATE_DIR/latest.env\\\"\"},{\"line\":8,\"text\":\"TMP=\\\"$(mktemp)\\\"\"},{\"line\":9,\"text\":\"\"},{\"line\":10,\"text\":\"cleanup() {\"},{\"line\":11,\"text\":\" rm -f \\\"$TMP\\\"\"},{\"line\":12,\"text\":\"}\"},{\"line\":13,\"text\":\"trap cleanup EXIT\"},{\"line\":14,\"text\":\"\"},{\"line\":15,\"text\":\"mapfile -t ACTIVATION_FILES < <(\"},{\"line\":16,\"text\":\" find \\\\\"},{\"line\":17,\"text\":\" /var/lib/homelab-cluster-admin \\\\\"},{\"line\":18,\"text\":\" /opt \\\\\"},{\"line\":19,\"text\":\" /srv \\\\\"},{\"line\":20,\"text\":\" /usr/local/share \\\\\"},{\"line\":21,\"text\":\" -type f \\\\\"},{\"line\":22,\"text\":\" -path '*/.stage4i-control/activation.env' \\\\\"},{\"line\":23,\"text\":\" -print 2>/dev/null |\"},{\"line\":24,\"text\":\" sort -u\"},{\"line\":25,\"text\":\")\"},{\"line\":26,\"text\":\"\"},{\"line\":27,\"text\":\"if [ \\\"${#ACTIVATION_FILES[@]}\\\" -ne 1 ]; then\"},{\"line\":28,\"text\":\" echo \\\"STAGE4I_VERIFICATION_STATUS=FAILED\\\"\"},{\"line\":29,\"text\":\" echo \\\"REASON=ACTIVATION_FILE_COUNT_${#ACTIVATION_FILES[@]}\\\"\"},{\"line\":30,\"text\":\" exit 50\"},{\"line\":31,\"text\":\"fi\"},{\"line\":32,\"text\":\"\"},{\"line\":33,\"text\":\"ACTIVATION_FILE=\\\"${ACTIVATION_FILES[0]}\\\"\"},{\"line\":34,\"text\":\"CONTROL_DIR=\\\"$(dirname \\\"$ACTIVATION_FILE\\\")\\\"\"},{\"line\":35,\"text\":\"ACTIVE_LINK=\\\"$CONTROL_DIR/active\\\"\"},{\"line\":36,\"text\":\"ROLLBACK_LINK=\\\"$CONTROL_DIR/rollback\\\"\"},{\"line\":37,\"text\":\"\"},{\"line\":38,\"text\":\"# shellcheck disable=SC1090\"},{\"line\":39,\"text\":\"source \\\"$ACTIVATION_FILE\\\"\"},{\"line\":40,\"text\":\"\"},{\"line\":41,\"text\":\"test \\\"${STAGE4I_ACTIVATION_STATUS:-}\\\" = \\\"ACTIVE\\\"\"},{\"line\":42,\"text\":\"test -L \\\"$ACTIVE_LINK\\\"\"},{\"line\":43,\"text\":\"test -L \\\"$ROLLBACK_LINK\\\"\"},{\"line\":44,\"text\":\"\"},{\"line\":45,\"text\":\"ACTIVE_RESOLVED=\\\"$(readlink -f \\\"$ACTIVE_LINK\\\")\\\"\"},{\"line\":46,\"text\":\"ROLLBACK_RESOLVED=\\\"$(readlink -f \\\"$ROLLBACK_LINK\\\")\\\"\"},{\"line\":47,\"text\":\"\"},{\"line\":48,\"text\":\"test \\\"$ACTIVE_RESOLVED\\\" = \\\"${ACTIVE_PATH:-}\\\"\"},{\"line\":49,\"text\":\"test \\\"$ROLLBACK_RESOLVED\\\" = \\\"${ROLLBACK_PATH:-}\\\"\"},{\"line\":50,\"text\":\"test -d \\\"$ACTIVE_RESOLVED\\\"\"},{\"line\":51,\"text\":\"test -d \\\"$ROLLBACK_RESOLVED\\\"\"},{\"line\":52,\"text\":\"\"},{\"line\":53,\"text\":\"META=\\\"$ACTIVE_RESOLVED/IMMUTABLE.env\\\"\"},{\"line\":54,\"text\":\"MANIFEST=\\\"$ACTIVE_RESOLVED/MANIFEST.sha256\\\"\"},{\"line\":55,\"text\":\"\"},{\"line\":56,\"text\":\"test -f \\\"$META\\\"\"},{\"line\":57,\"text\":\"test -f \\\"$MANIFEST\\\"\"},{\"line\":58,\"text\":\"grep -q \\\"^BUILD_MARKER=$MARKER$\\\" \\\"$META\\\"\"},{\"line\":59,\"text\":\"grep -q '^TASK_STAGE=4I$' \\\"$META\\\"\"},{\"line\":60,\"text\":\"grep -q '^TASK_STATUS=IMMUTABLE$' \\\"$META\\\"\"},{\"line\":61,\"text\":\"\"},{\"line\":62,\"text\":\"(\"},{\"line\":63,\"text\":\" cd \\\"$ACTIVE_RESOLVED\\\"\"},{\"line\":64,\"text\":\" sha256sum -c MANIFEST.sha256 >/dev/null\"},{\"line\":65,\"text\":\")\"},{\"line\":66,\"text\":\"\"},{\"line\":67,\"text\":\"if find \\\"$ACTIVE_RESOLVED\\\" \\\\\"},{\"line\":68,\"text\":\" \\\\( ! -user root -o ! -group root \\\\) \\\\\"},{\"line\":69,\"text\":\" -print -quit |\"},{\"line\":70,\"text\":\" grep -q .\"},{\"line\":71,\"text\":\"then\"},{\"line\":72,\"text\":\" echo \\\"STAGE4I_VERIFICATION_STATUS=FAILED\\\"\"},{\"line\":73,\"text\":\" echo \\\"REASON=INVALID_OWNER\\\"\"},{\"line\":74,\"text\":\" exit 51\"},{\"line\":75,\"text\":\"fi\"},{\"line\":76,\"text\":\"\"},{\"line\":77,\"text\":\"if find \\\"$ACTIVE_RESOLVED\\\" \\\\\"},{\"line\":78,\"text\":\" -perm /0222 \\\\\"},{\"line\":79,\"text\":\" -print -quit |\"},{\"line\":80,\"text\":\" grep -q .\"},{\"line\":81,\"text\":\"then\"},{\"line\":82,\"text\":\" echo \\\"STAGE4I_VERIFICATION_STATUS=FAILED\\\"\"},{\"line\":83,\"text\":\" echo \\\"REASON=WRITABLE_IMMUTABLE_CONTENT\\\"\"},{\"line\":84,\"text\":\" exit 52\"},{\"line\":85,\"text\":\"fi\"},{\"line\":86,\"text\":\"\"},{\"line\":87,\"text\":\"install -d -m 0750 -o root -g root \\\"$STATE_DIR\\\"\"},{\"line\":88,\"text\":\"\"},{\"line\":89,\"text\":\"{\"},{\"line\":90,\"text\":\" echo \\\"STAGE4I_VERIFICATION_STATUS=SUCCESS\\\"\"},{\"line\":91,\"text\":\" echo \\\"VERIFIED_AT_UTC=$(date -u +%Y-%m-%dT%H:%M:%SZ)\\\"\"},{\"line\":92,\"text\":\" echo \\\"ACTIVE_VERSION=${ACTIVE_VERSION:-UNKNOWN}\\\"\"},{\"line\":93,\"text\":\" echo \\\"ACTIVE_PATH=$ACTIVE_RESOLVED\\\"\"},{\"line\":94,\"text\":\" echo \\\"ROLLBACK_VERSION=${ROLLBACK_VERSION:-UNKNOWN}\\\"\"},{\"line\":95,\"text\":\" echo \\\"ROLLBACK_PATH=$ROLLBACK_RESOLVED\\\"\"},{\"line\":96,\"text\":\" echo \\\"MANIFEST_SHA256=$(sha256sum \\\"$MANIFEST\\\" | awk '{print $1}')\\\"\"},{\"line\":97,\"text\":\" echo \\\"VERIFIED_ON_HOST=$(hostname -f 2>/dev/null || hostname)\\\"\"},{\"line\":98,\"text\":\"} >\\\"$TMP\\\"\"},{\"line\":99,\"text\":\"\"},{\"line\":100,\"text\":\"chown root:root \\\"$TMP\\\"\"},{\"line\":101,\"text\":\"chmod 0440 \\\"$TMP\\\"\"},{\"line\":102,\"text\":\"mv -f \\\"$TMP\\\" \\\"$REPORT\\\"\"},{\"line\":103,\"text\":\"trap - EXIT\"},{\"line\":104,\"text\":\"\"},{\"line\":105,\"text\":\"cat \\\"$REPORT\\\"\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":5,\"text\":\"Requires=postgresql.service\"},{\"line\":7,\"text\":\"[Service]\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"binary\":null,\"bytes\":616,\"line_count\":19,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"Requires=postgresql.service\"},{\"line\":6,\"text\":\"\"},{\"line\":7,\"text\":\"[Service]\"},{\"line\":8,\"text\":\"Type=oneshot\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":13,\"text\":\"UMask=0027\"},{\"line\":14,\"text\":\"NoNewPrivileges=true\"},{\"line\":15,\"text\":\"PrivateTmp=true\"},{\"line\":16,\"text\":\"ProtectSystem=strict\"},{\"line\":17,\"text\":\"ProtectHome=true\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"binary\":null,\"bytes\":231,\"line_count\":12,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=30s\"},{\"line\":6,\"text\":\"OnUnitActiveSec=60s\"},{\"line\":7,\"text\":\"AccuracySec=10s\"},{\"line\":8,\"text\":\"Persistent=true\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"},{\"line\":10,\"text\":\"\"},{\"line\":11,\"text\":\"[Install]\"},{\"line\":12,\"text\":\"WantedBy=timers.target\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service\"},{\"line\":5,\"text\":\"Requires=postgresql.service\"},{\"line\":7,\"text\":\"[Service]\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"binary\":null,\"bytes\":803,\"line_count\":28,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"Requires=postgresql.service\"},{\"line\":6,\"text\":\"\"},{\"line\":7,\"text\":\"[Service]\"},{\"line\":8,\"text\":\"Type=simple\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":13,\"text\":\"ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\"},{\"line\":14,\"text\":\"Restart=on-failure\"},{\"line\":15,\"text\":\"RestartSec=5\"},{\"line\":16,\"text\":\"UMask=0027\"},{\"line\":17,\"text\":\"NoNewPrivileges=true\"},{\"line\":18,\"text\":\"PrivateTmp=true\"},{\"line\":19,\"text\":\"ProtectSystem=strict\"},{\"line\":20,\"text\":\"ProtectHome=true\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"},{\"line\":23,\"text\":\"RestrictSUIDSGID=true\"},{\"line\":24,\"text\":\"LockPersonality=true\"},{\"line\":25,\"text\":\"RestrictRealtime=true\"},{\"line\":26,\"text\":\"\"},{\"line\":27,\"text\":\"[Install]\"},{\"line\":28,\"text\":\"WantedBy=multi-user.target\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":6,\"text\":\"[Service]\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"binary\":null,\"bytes\":1190,\"line_count\":42,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":3,\"text\":\"After=network-online.target\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"\"},{\"line\":6,\"text\":\"[Service]\"},{\"line\":7,\"text\":\"Type=oneshot\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":12,\"text\":\"UMask=0077\"},{\"line\":13,\"text\":\"NoNewPrivileges=yes\"},{\"line\":14,\"text\":\"PrivateTmp=yes\"},{\"line\":15,\"text\":\"PrivateDevices=yes\"},{\"line\":16,\"text\":\"ProtectSystem=strict\"},{\"line\":17,\"text\":\"ProtectHome=yes\"},{\"line\":18,\"text\":\"ProtectKernelTunables=yes\"},{\"line\":19,\"text\":\"ProtectKernelModules=yes\"},{\"line\":20,\"text\":\"ProtectKernelLogs=yes\"},{\"line\":21,\"text\":\"ProtectControlGroups=yes\"},{\"line\":22,\"text\":\"ProtectClock=yes\"},{\"line\":23,\"text\":\"ProtectHostname=yes\"},{\"line\":24,\"text\":\"LockPersonality=yes\"},{\"line\":25,\"text\":\"MemoryDenyWriteExecute=yes\"},{\"line\":26,\"text\":\"RestrictSUIDSGID=yes\"},{\"line\":27,\"text\":\"RestrictRealtime=yes\"},{\"line\":28,\"text\":\"RemoveIPC=yes\"},{\"line\":29,\"text\":\"CapabilityBoundingSet=\"},{\"line\":30,\"text\":\"AmbientCapabilities=\"},{\"line\":31,\"text\":\"RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\"},{\"line\":32,\"text\":\"IPAddressDeny=any\"},{\"line\":33,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":34,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":35,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":36,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":40,\"text\":\"StateDirectoryMode=0750\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"},{\"line\":42,\"text\":\"TimeoutStartSec=70\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"binary\":null,\"bytes\":223,\"line_count\":12,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=2min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=60s\"},{\"line\":7,\"text\":\"RandomizedDelaySec=5s\"},{\"line\":8,\"text\":\"Persistent=true\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"},{\"line\":10,\"text\":\"\"},{\"line\":11,\"text\":\"[Install]\"},{\"line\":12,\"text\":\"WantedBy=timers.target\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"binary\":null,\"bytes\":133,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"binary\":null,\"bytes\":154,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=5min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=5min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"binary\":null,\"bytes\":131,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"binary\":null,\"bytes\":149,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=3min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=5min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"binary\":null,\"bytes\":127,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}]},{\"anchors\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"binary\":null,\"bytes\":151,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"priority\":90,\"role\":\"unit-fragment\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=2min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=5min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]}],\"transaction_gate\":{\"frontend_index_exact\":true,\"ready_for_final_atomic_design\":true,\"reference_backend_exact\":true,\"remote_root_probe\":true,\"selected_source_count\":18,\"unit_count\":16},\"working_directories\":[\"/opt/cluster-admin-incident-engine\"]},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":19,\"status\":\"OK\",\"v18_exact_result\":{\"capsule_sha256\":\"59b7a04f38f9ccf08b499fea9082c9864bebf360c6f33d25502e3811ab762db6\",\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z\",\"control_plane_error\":{\"message\":\"existing runner returned a different rc after writing its public artifact; runtime publication or wrapper completion failed\",\"type\":\"RunnerWrapperMismatch\"},\"declared_report_bytes\":193466,\"declared_report_sha256\":\"772f2c3f77d7909de34aa159e0e2c8c2ca5480ff51cbcd4bcb194af9019b43d0\",\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":194314,\"output_sha256\":\"5b17a087bd51a9f4e901cd820e129160e70081636d2da63d902f481305df4f99\",\"output_source\":\"runner_json\",\"rollback_restored\":null,\"rollback_started\":false,\"runner_document_rc\":0,\"runner_json_sha256\":\"a73b7ed1a9bca8aac9fe10824f3921f6a45e1898091a9f8caafdd3dc699f28d7\",\"runner_rc\":75,\"runner_status\":\"OK\",\"runner_text_sha256\":\"924eb43f0b1633f8b40a49cae8d04af57f9fbd82e61acc8d083c2e3050d7ba6f\",\"runner_wrapper_matches_document\":false,\"sanitized_report_bytes\":193386,\"sanitized_report_sha256\":\"2a7d38c6c1a684e26469ec611090b0916fbefb54f1621f84dbe3ae6672c599bb\",\"state_document_sha256\":\"212ada5f20b26b3a7b1044ef597d7562c83666f391f1b9cbb0d91bf899e9a95d\",\"state_path_sha256\":\"cc9ddb1c213dedb6c87405dd161acb5b8cd92b28cfdf6a153a03e7728ce9e456\",\"state_status\":\"FAIL\"}}\n"
}
View Git Blame Copy Permalink