homelab-public-context/runtime/latest.json

{
  "schema_version": 1,
  "channel": "homelab-runtime",
  "command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z",
  "status": "OK",
  "rc": 0,
  "host": "pve01",
  "mode": "verify",
  "component": "homelab-cluster-admin-update-visibility",
  "started_at_utc": "2026-08-05T13:20:59Z",
  "finished_at_utc": "2026-08-05T13:20:59Z",
  "reference_register_checked": true,
  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
  "error_register_checked": true,
  "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
  "command_sha256": "8e9116b7bd5174aea8762e6a52c7c3efb4eb6ea1c0e202c3bae875aaead38778",
  "duplicate_failed_command_blocked": false,
  "block_reason": null,
  "execution_started": true,
  "change_declared": false,
  "result_contract_valid": true,
  "result_contract_status": null,
  "result_contract_error": null,
  "command_rc": 0,
  "changes_made": false,
  "rollback_started": false,
  "rollback_restored": null,
  "mutation_outcome": "NO_MUTATION",
  "sanitized": true,
  "secrets_included": false,
  "private_addresses_included": false,
  "raw_evidence_retained_locally": true,
  "raw_evidence_sha256": "7789c8f8591d0be49d972a2812151287eb12642bd99740576b1b9d9a29f84267",
  "sanitized_output_sha256": "5b17a087bd51a9f4e901cd820e129160e70081636d2da63d902f481305df4f99",
  "output_truncated_in_json": false,
  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z\",\"command_rc\":0,\"control_plane_error\":null,\"exact_source_count\":30,\"index_source_count\":1,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_atomic_transaction\":true,\"reference_source_count\":1,\"remote_probe_privilege\":\"sudo-root\",\"report_bytes\":193466,\"report_sha256\":\"772f2c3f77d7909de34aa159e0e2c8c2ca5480ff51cbcd4bcb194af9019b43d0\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"unit_record_count\":16,\"v15_declared_report_sha256\":\"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf\",\"v15_exact_result_identity_confirmed\":true,\"v15_output_sha256\":\"c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8\",\"v15_sanitized_report_sha256\":\"f05595551c37c00d473df29113cf9d597b2d3467ad5a2b50e8803e8579accc82\",\"v15_sanitizer_byte_delta\":50,\"version\":1}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z\",\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"remote_exact_map\":{\"euid\":0,\"exact_manifest\":[{\"bytes\":3708,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"priority\":100,\"requested_as\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"uid\":0},{\"bytes\":10353,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"priority\":95,\"requested_as\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"uid\":0},{\"bytes\":2832,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"priority\":92,\"requested_as\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\",\"uid\":0},{\"bytes\":3471,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"priority\":92,\"requested_as\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\",\"uid\":0},{\"bytes\":1433,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"priority\":92,\"requested_as\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\",\"uid\":0},{\"bytes\":848,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"priority\":92,\"requested_as\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\",\"uid\":0},{\"error\":\"RuntimeError: unsafe file: /usr/bin/python3.11\",\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"path\":\"/usr/bin/python3.11\",\"priority\":92,\"requested_as\":\"/usr/bin/python3\"},{\"bytes\":2541,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/usr/local/sbin/homelab-stage4i-verify\",\"priority\":92,\"requested_as\":\"/usr/local/sbin/homelab-stage4i-verify\",\"sha256\":\"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d\",\"uid\":0},{\"bytes\":616,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\",\"uid\":0},{\"bytes\":231,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\",\"uid\":0},{\"bytes\":803,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\",\"uid\":0},{\"bytes\":1190,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\",\"uid\":0},{\"bytes\":223,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\",\"uid\":0},{\"bytes\":133,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\",\"uid\":0},{\"bytes\":154,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\",\"uid\":0},{\"bytes\":131,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\",\"uid\":0},{\"bytes\":149,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\",\"uid\":0},{\"bytes\":127,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\",\"uid\":0},{\"bytes\":151,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\",\"uid\":0},{\"bytes\":138,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\",\"uid\":0},{\"bytes\":154,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\",\"uid\":0},{\"bytes\":301,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"uid\":0},{\"bytes\":465,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"sha256\":\"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b\",\"uid\":0},{\"bytes\":224,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"sha256\":\"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396\",\"uid\":0},{\"is_dir\":true,\"is_file\":false,\"is_symlink\":false,\"path\":\"/opt/cluster-admin-incident-engine\",\"priority\":75,\"requested_as\":\"/opt/cluster-admin-incident-engine\"},{\"bytes\":380,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"priority\":70,\"requested_as\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\",\"uid\":0},{\"bytes\":217,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"priority\":70,\"requested_as\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\",\"uid\":0},{\"bytes\":128,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"priority\":70,\"requested_as\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\",\"uid\":0},{\"bytes\":1535,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"priority\":70,\"requested_as\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\",\"uid\":0},{\"bytes\":616,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\",\"uid\":0},{\"bytes\":231,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\",\"uid\":0},{\"bytes\":803,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\",\"uid\":0},{\"bytes\":1190,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\",\"uid\":0},{\"bytes\":223,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\",\"uid\":0},{\"bytes\":133,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\",\"uid\":0},{\"bytes\":154,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\",\"uid\":0},{\"bytes\":131,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\",\"uid\":0},{\"bytes\":149,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\",\"uid\":0},{\"bytes\":127,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\",\"uid\":0},{\"bytes\":151,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\",\"uid\":0},{\"bytes\":138,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\",\"uid\":0},{\"bytes\":154,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\",\"uid\":0},{\"bytes\":301,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"uid\":0},{\"bytes\":465,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b\",\"uid\":0},{\"bytes\":224,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396\",\"uid\":0},{\"bytes\":2832,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\",\"uid\":0},{\"bytes\":3471,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\",\"uid\":0},{\"bytes\":1433,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\",\"uid\":0},{\"bytes\":3708,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"uid\":0},{\"bytes\":848,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\",\"uid\":0},{\"bytes\":10353,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"uid\":0},{\"bytes\":89,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"priority\":50,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\",\"uid\":0},{\"bytes\":28599,\"gid\":994,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\",\"uid\":0},{\"bytes\":10640,\"gid\":994,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\",\"uid\":0},{\"bytes\":1857,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\",\"uid\":0},{\"bytes\":2355,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\",\"uid\":0},{\"bytes\":1342,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\",\"uid\":0},{\"bytes\":922,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\",\"uid\":0},{\"bytes\":8444,\"gid\":994,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\",\"uid\":0},{\"bytes\":131,\"gid\":994,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"priority\":50,\"requested_as\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"5348849c615d0258d7812fdcf660611f911b97753bbab2e0fd87e5b1c2a98cf8\",\"uid\":999},{\"bytes\":259,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"275b09782748c2b8a1b762b6390a84df202e478ece7e8f377788569b2cc5b027\",\"uid\":0},{\"bytes\":168,\"gid\":994,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"5b2bc5ae640f3b7b42000bae6b5b63d76ba00a8f55f4b6cc1cc9f80871373653\",\"uid\":999},{\"bytes\":189,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\",\"uid\":0},{\"bytes\":141,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\",\"uid\":0},{\"bytes\":199,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\",\"uid\":0},{\"bytes\":272,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"49dbf42b944b78c7394d299a08400d24a92b7532fe8d9774c686fe6a59972e15\",\"uid\":0},{\"bytes\":134,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"6353b738b98d4d038684bc753014410f96cf0ace91227dad461b4a2c69ee3c29\",\"uid\":0},{\"bytes\":214,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"02916ca8a08ee305709d7fa7935209a6321b0e25fb105e0f2d3e242ed66fb0f0\",\"uid\":0},{\"bytes\":214,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"priority\":50,\"requested_as\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"02916ca8a08ee305709d7fa7935209a6321b0e25fb105e0f2d3e242ed66fb0f0\",\"uid\":0}],\"exact_sources\":[{\"bytes\":3708,\"line_count\":79,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"priority\":100,\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":5,\"text\":\"mkdir -p \\\"$WEB\\\"\"},{\"line\":6,\"text\":\"BAD=0\"},{\"line\":7,\"text\":\"\"},{\"line\":8,\"text\":\"line_or_missing() {\"},{\"line\":9,\"text\":\"  f=\\\"$1\\\"\"},{\"line\":10,\"text\":\"  if [ -s \\\"$f\\\" ]; then head -n1 \\\"$f\\\"; else echo \\\"STATUS=WARN TYPE=missing FILE=$f\\\"; fi\"},{\"line\":11,\"text\":\"}\"},{\"line\":12,\"text\":\"\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":20,\"text\":\"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":21,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":22,\"text\":\"\"},{\"line\":23,\"text\":\"for line in \\\"$SELF\\\" \\\"$OBSERVER\\\" \\\"$FULL\\\" \\\"$BACKUP\\\"; do\"},{\"line\":24,\"text\":\"  echo \\\"$line\\\" | grep -q '^STATUS=OK' || BAD=$((BAD+1))\"},{\"line\":25,\"text\":\"done\"},{\"line\":26,\"text\":\"echo \\\"$FULL\\\" | grep -q 'BAD=0' || BAD=$((BAD+1))\"},{\"line\":27,\"text\":\"echo \\\"$FULL\\\" | grep -q 'WARN=0' || BAD=$((BAD+1))\"},{\"line\":28,\"text\":\"echo \\\"$BACKUP\\\" | grep -q 'BAD=0' || BAD=$((BAD+1))\"},{\"line\":29,\"text\":\"echo \\\"$BACKUP\\\" | grep -q 'MAIL_CLOUD_2COPY_VERIFY_OK' || BAD=$((BAD+1))\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":31,\"text\":\"[ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":32,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":33,\"text\":\"\"},{\"line\":34,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; COLOR=\\\"#0a7f38\\\"; BADGE=\\\"OK\\\"; else STATUS=WARN; COLOR=\\\"#b26a00\\\"; BADGE=\\\"REVIEW\\\"; fi\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":37,\"text\":\"echo \\\"$STATUS_LINE\\\" > \\\"$WEB/status.txt\\\"\"},{\"line\":38,\"text\":\"\"},{\"line\":39,\"text\":\"cat > \\\"$WEB/index.html\\\" <<HTML\"},{\"line\":40,\"text\":\"<!doctype html>\"},{\"line\":41,\"text\":\"<html lang=\\\"ru\\\">\"},{\"line\":42,\"text\":\"<head>\"},{\"line\":43,\"text\":\"<meta charset=\\\"utf-8\\\">\"},{\"line\":44,\"text\":\"<meta http-equiv=\\\"refresh\\\" content=\\\"60\\\">\"},{\"line\":45,\"text\":\"<meta name=\\\"viewport\\\" content=\\\"width=device-width, initial-scale=1\\\">\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":47,\"text\":\"<style>\"},{\"line\":48,\"text\":\"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}\"},{\"line\":49,\"text\":\".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}\"},{\"line\":50,\"text\":\"h1{margin:0 0 8px}\"},{\"line\":51,\"text\":\".badge{display:inline-block;background:$COLOR;color:white;padding:8px 14px;border-radius:999px;font-weight:700}\"},{\"line\":52,\"text\":\"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}\"},{\"line\":53,\"text\":\".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}\"},{\"line\":54,\"text\":\".small{color:#aab2c0}\"},{\"line\":55,\"text\":\"</style>\"},{\"line\":56,\"text\":\"</head>\"},{\"line\":57,\"text\":\"<body>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":59,\"text\":\"<div class=\\\"small\\\">generated UTC: $TS · VM180 · [PRIVATE_IP] · refresh 60s</div>\"},{\"line\":60,\"text\":\"<p><span class=\\\"badge\\\">$BADGE</span></p>\"},{\"line\":61,\"text\":\"\"},{\"line\":62,\"text\":\"<div class=\\\"grid\\\">\"},{\"line\":63,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>$SELF\"},{\"line\":64,\"text\":\"FAILED_UNITS=$FAILED\"},{\"line\":65,\"text\":\"QGA=$QGA</pre></div>\"},{\"line\":66,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>$OBSERVER</pre></div>\"},{\"line\":67,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>$FULL</pre></div>\"},{\"line\":68,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>$BACKUP</pre></div>\"},{\"line\":69,\"text\":\"</div>\"},{\"line\":70,\"text\":\"\"},{\"line\":71,\"text\":\"<div class=\\\"card\\\"><h2>Overall</h2><pre>$OVERALL</pre></div>\"},{\"line\":72,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>$FINAL</pre></div>\"},{\"line\":73,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>$STRICT</pre></div>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":75,\"text\":\"</body>\"},{\"line\":76,\"text\":\"</html>\"},{\"line\":77,\"text\":\"HTML\"},{\"line\":78,\"text\":\"\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}]},{\"bytes\":10353,\"line_count\":80,\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"priority\":95,\"sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"source\":[{\"line\":1,\"text\":\"<!doctype html>\"},{\"line\":2,\"text\":\"<html lang=\\\"ru\\\">\"},{\"line\":3,\"text\":\"<head>\"},{\"line\":4,\"text\":\"<meta charset=\\\"utf-8\\\">\"},{\"line\":5,\"text\":\"<meta http-equiv=\\\"refresh\\\" content=\\\"60\\\">\"},{\"line\":6,\"text\":\"<meta name=\\\"viewport\\\" content=\\\"width=device-width, initial-scale=1\\\">\"},{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":8,\"text\":\"<style>\"},{\"line\":9,\"text\":\"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}\"},{\"line\":10,\"text\":\".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}\"},{\"line\":11,\"text\":\"h1{margin:0 0 8px}\"},{\"line\":12,\"text\":\".badge{display:inline-block;background:#b26a00;color:white;padding:8px 14px;border-radius:999px;font-weight:700}\"},{\"line\":13,\"text\":\"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}\"},{\"line\":14,\"text\":\".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}\"},{\"line\":15,\"text\":\".small{color:#aab2c0}\"},{\"line\":16,\"text\":\"</style>\"},{\"line\":17,\"text\":\"</head>\"},{\"line\":18,\"text\":\"<body>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":20,\"text\":\"<div class=\\\"small\\\">generated UTC: 2026-08-05T13:20:48Z · VM180 · [PRIVATE_IP] · refresh 60s</div>\"},{\"line\":21,\"text\":\"<p><span class=\\\"badge\\\">REVIEW</span></p>\"},{\"line\":22,\"text\":\"\"},{\"line\":23,\"text\":\"<div class=\\\"grid\\\">\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":25,\"text\":\"FAILED_UNITS=1\"},{\"line\":26,\"text\":\"QGA=active</pre></div>\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":30,\"text\":\"</div>\"},{\"line\":31,\"text\":\"\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":36,\"text\":\"generated_utc=2026-08-05T13:19:08Z\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":38,\"text\":\"\"},{\"line\":39,\"text\":\"## Restricted probes\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":44,\"text\":\"\"},{\"line\":45,\"text\":\"## Service map checks\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":47,\"text\":\"OK severity=critical kind=tcp name=pve01_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":48,\"text\":\"OK severity=critical kind=tcp name=pve01_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\"},{\"line\":49,\"text\":\"OK severity=critical kind=tcp name=pve01_health target=[PRIVATE_IP] value=9101 tcp_9101_rc=0\"},{\"line\":50,\"text\":\"OK severity=critical kind=tcp name=pve02_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":51,\"text\":\"OK severity=critical kind=tcp name=pve02_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\"},{\"line\":52,\"text\":\"OK severity=critical kind=tcp name=pve03_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":53,\"text\":\"OK severity=critical kind=tcp name=pve03_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\"},{\"line\":54,\"text\":\"OK severity=critical kind=tcp name=dns1_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0\"},{\"line\":55,\"text\":\"OK severity=critical kind=tcp name=dns2_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0\"},{\"line\":56,\"text\":\"OK severity=critical kind=tcp name=edge_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":58,\"text\":\"OK severity=critical kind=ping name=vm130_edge_vm_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":59,\"text\":\"OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":60,\"text\":\"OK severity=critical kind=ping name=vm150_nextcloud_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":61,\"text\":\"OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":62,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\"},{\"line\":63,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":64,\"text\":\"OK severity=critical kind=ping name=vm160_forum_prod_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":65,\"text\":\"OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":66,\"text\":\"OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=0\"},{\"line\":67,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":68,\"text\":\"OK severity=critical kind=ping name=vm170_core_apps_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":69,\"text\":\"OK severity=critical kind=tcp name=vm170_core_apps_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":70,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\"},{\"line\":71,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":72,\"text\":\"OK severity=critical kind=ping name=vm171_monitoring_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":73,\"text\":\"OK severity=critical kind=tcp name=vm171_monitoring_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":74,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\"},{\"line\":75,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":76,\"text\":\"\"},{\"line\":77,\"text\":\"## Result\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":79,\"text\":\"</body>\"},{\"line\":80,\"text\":\"</html>\"}]},{\"bytes\":2832,\"line_count\":96,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"priority\":92,\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":4,\"text\":\"exec 9>\\\"$LOCK\\\"\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":6,\"text\":\"\"},{\"line\":7,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":10,\"text\":\"REPORT=\\\"$ROOT/latest.md\\\"\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":12,\"text\":\"TMP=\\\"$REPORT.tmp\\\"\"},{\"line\":13,\"text\":\"mkdir -p \\\"$ROOT\\\" \\\"$HEALTH_DIR\\\"\"},{\"line\":14,\"text\":\"BAD=0\"},{\"line\":15,\"text\":\"WARN=0\"},{\"line\":16,\"text\":\"\"},{\"line\":17,\"text\":\"check_ping() {\"},{\"line\":18,\"text\":\"  name=\\\"$1\\\"; ip=\\\"$2\\\"\"},{\"line\":19,\"text\":\"  ping -c1 -W1 \\\"$ip\\\" >/dev/null 2>&1\"},{\"line\":20,\"text\":\"  rc=$?\"},{\"line\":21,\"text\":\"  echo \\\"$name ping rc=$rc\\\" >> \\\"$TMP\\\"\"},{\"line\":22,\"text\":\"  [ \\\"$rc\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":23,\"text\":\"}\"},{\"line\":24,\"text\":\"\"},{\"line\":25,\"text\":\"check_tcp() {\"},{\"line\":26,\"text\":\"  name=\\\"$1\\\"; ip=\\\"$2\\\"; port=\\\"$3\\\"\"},{\"line\":27,\"text\":\"  timeout 2 bash -c \\\"</dev/tcp/$ip/$port\\\" >/dev/null 2>&1\"},{\"line\":28,\"text\":\"  rc=$?\"},{\"line\":29,\"text\":\"  echo \\\"$name tcp/$port rc=$rc\\\" >> \\\"$TMP\\\"\"},{\"line\":30,\"text\":\"  [ \\\"$rc\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":31,\"text\":\"}\"},{\"line\":32,\"text\":\"\"},{\"line\":33,\"text\":\"check_http_status_ok() {\"},{\"line\":34,\"text\":\"  name=\\\"$1\\\"; url=\\\"$2\\\"\"},{\"line\":35,\"text\":\"  line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":36,\"text\":\"  rc=$?\"},{\"line\":37,\"text\":\"  echo \\\"$name http rc=$rc line=$line\\\" >> \\\"$TMP\\\"\"},{\"line\":38,\"text\":\"  echo \\\"$line\\\" | grep -q \\\"^STATUS=\\\"\"},{\"line\":39,\"text\":\"  okrc=$?\"},{\"line\":40,\"text\":\"  [ \\\"$rc\\\" = 0 ] && [ \\\"$okrc\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":41,\"text\":\"}\"},{\"line\":42,\"text\":\"\"},{\"line\":43,\"text\":\"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":44,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":46,\"text\":\"\"},{\"line\":47,\"text\":\"{\"},{\"line\":48,\"text\":\"  echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":49,\"text\":\"  echo \\\"generated_utc=$TS\\\"\"},{\"line\":50,\"text\":\"  echo\"},{\"line\":51,\"text\":\"  echo \\\"## Self\\\"\"},{\"line\":52,\"text\":\"  echo \\\"host=$(hostname 2>/dev/null)\\\"\"},{\"line\":53,\"text\":\"  echo \\\"failed_units=$FAILED\\\"\"},{\"line\":54,\"text\":\"  echo \\\"qga=$QGA\\\"\"},{\"line\":55,\"text\":\"  echo \\\"self_check=$SELF_LINE\\\"\"},{\"line\":56,\"text\":\"  echo\"},{\"line\":57,\"text\":\"  echo \\\"## Network checks\\\"\"},{\"line\":58,\"text\":\"} > \\\"$TMP\\\"\"},{\"line\":59,\"text\":\"\"},{\"line\":60,\"text\":\"[ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":61,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":62,\"text\":\"echo \\\"$SELF_LINE\\\" | grep -q \\\"^STATUS=OK\\\" || BAD=$((BAD+1))\"},{\"line\":63,\"text\":\"\"},{\"line\":64,\"text\":\"check_ping pve01 [PRIVATE_IP]\"},{\"line\":65,\"text\":\"check_tcp pve01_ssh [PRIVATE_IP] 22\"},{\"line\":66,\"text\":\"check_tcp pve01_proxmox [PRIVATE_IP] 8006\"},{\"line\":67,\"text\":\"check_tcp pve01_health [PRIVATE_IP] 9101\"},{\"line\":68,\"text\":\"\"},{\"line\":69,\"text\":\"check_ping pve02 [PRIVATE_IP]\"},{\"line\":70,\"text\":\"check_tcp pve02_ssh [PRIVATE_IP] 22\"},{\"line\":71,\"text\":\"check_tcp pve02_proxmox [PRIVATE_IP] 8006\"},{\"line\":72,\"text\":\"\"},{\"line\":73,\"text\":\"check_ping pve03 [PRIVATE_IP]\"},{\"line\":74,\"text\":\"check_tcp pve03_ssh [PRIVATE_IP] 22\"},{\"line\":75,\"text\":\"check_tcp pve03_proxmox [PRIVATE_IP] 8006\"},{\"line\":76,\"text\":\"\"},{\"line\":77,\"text\":\"check_ping edge [PRIVATE_IP]\"},{\"line\":78,\"text\":\"check_tcp edge_ssh [PRIVATE_IP] 22\"},{\"line\":79,\"text\":\"\"},{\"line\":80,\"text\":\"check_ping dns1 [PRIVATE_IP]\"},{\"line\":81,\"text\":\"check_tcp dns1_tcp53 [PRIVATE_IP] 53\"},{\"line\":82,\"text\":\"check_ping dns2 [PRIVATE_IP]\"},{\"line\":83,\"text\":\"check_tcp dns2_tcp53 [PRIVATE_IP] 53\"},{\"line\":84,\"text\":\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":86,\"text\":\"\"},{\"line\":87,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"},{\"line\":89,\"text\":\"echo \\\"$LINE\\\" | tee \\\"$HEALTH\\\" >/dev/null\"},{\"line\":90,\"text\":\"{\"},{\"line\":91,\"text\":\"  echo\"},{\"line\":92,\"text\":\"  echo \\\"## Result\\\"\"},{\"line\":93,\"text\":\"  echo \\\"$LINE\\\"\"},{\"line\":94,\"text\":\"} >> \\\"$TMP\\\"\"},{\"line\":95,\"text\":\"mv \\\"$TMP\\\" \\\"$REPORT\\\"\"},{\"line\":96,\"text\":\"cat \\\"$HEALTH\\\"\"}]},{\"bytes\":3471,\"line_count\":113,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"priority\":92,\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":7,\"text\":\"REPORT=\\\"$ROOT/full-observer-latest.md\\\"\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":10,\"text\":\"mkdir -p \\\"$ROOT\\\" \\\"$HEALTH_DIR\\\"\"},{\"line\":11,\"text\":\"exec 9>\\\"$LOCK\\\"\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":13,\"text\":\"\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":15,\"text\":\"BAD=0\"},{\"line\":16,\"text\":\"WARN=0\"},{\"line\":17,\"text\":\"INFO_OFF=0\"},{\"line\":18,\"text\":\"TOTAL=0\"},{\"line\":19,\"text\":\"CRIT_FAIL=0\"},{\"line\":20,\"text\":\"OPT_OFF=0\"},{\"line\":21,\"text\":\"PROBE_OK=0\"},{\"line\":22,\"text\":\"PROBE_BAD=0\"},{\"line\":23,\"text\":\"\"},{\"line\":24,\"text\":\"TMP=\\\"$REPORT.tmp\\\"\"},{\"line\":25,\"text\":\"{\"},{\"line\":26,\"text\":\"  echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":27,\"text\":\"  echo \\\"generated_utc=$TS\\\"\"},{\"line\":28,\"text\":\"  echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":29,\"text\":\"  echo\"},{\"line\":30,\"text\":\"  echo \\\"## Restricted probes\\\"\"},{\"line\":31,\"text\":\"} > \\\"$TMP\\\"\"},{\"line\":32,\"text\":\"\"},{\"line\":33,\"text\":\"for t in \\\"root@[PRIVATE_IP] pve01\\\" \\\"root@[PRIVATE_IP] pve02\\\" \\\"root@[PRIVATE_IP] pve03\\\" \\\"debian@[PRIVATE_IP] edge\\\"; do\"},{\"line\":34,\"text\":\"  set -- $t\"},{\"line\":35,\"text\":\"  target=\\\"$1\\\"; name=\\\"$2\\\"\"},{\"line\":36,\"text\":\"  out=$(ssh -n -i \\\"$K\\\" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 \\\"$target\\\" probe 2>/tmp/full-probe-$name.err)\"},{\"line\":37,\"text\":\"  rc=$?\"},{\"line\":38,\"text\":\"  echo \\\"$name ssh_rc=$rc $out\\\" >> \\\"$TMP\\\"\"},{\"line\":39,\"text\":\"  echo \\\"$out\\\" | grep -q '^STATUS=OK'\"},{\"line\":40,\"text\":\"  ok=$?\"},{\"line\":41,\"text\":\"  if [ \\\"$rc\\\" = 0 ] && [ \\\"$ok\\\" = 0 ]; then PROBE_OK=$((PROBE_OK+1)); else PROBE_BAD=$((PROBE_BAD+1)); BAD=$((BAD+1)); fi\"},{\"line\":42,\"text\":\"done\"},{\"line\":43,\"text\":\"\"},{\"line\":44,\"text\":\"{\"},{\"line\":45,\"text\":\"  echo\"},{\"line\":46,\"text\":\"  echo \\\"## Service map checks\\\"\"},{\"line\":47,\"text\":\"} >> \\\"$TMP\\\"\"},{\"line\":48,\"text\":\"\"},{\"line\":49,\"text\":\"while IFS=\\\"$(printf '\\\\t')\\\" read -r severity kind name target value; do\"},{\"line\":50,\"text\":\"  [ -n \\\"$severity\\\" ] || continue\"},{\"line\":51,\"text\":\"  TOTAL=$((TOTAL+1))\"},{\"line\":52,\"text\":\"  rc=0\"},{\"line\":53,\"text\":\"  detail=\\\"\\\"\"},{\"line\":54,\"text\":\"  case \\\"$kind\\\" in\"},{\"line\":55,\"text\":\"    ping)\"},{\"line\":56,\"text\":\"      ping -c1 -W1 \\\"$target\\\" >/dev/null 2>&1\"},{\"line\":57,\"text\":\"      rc=$?\"},{\"line\":58,\"text\":\"      detail=\\\"ping_rc=$rc\\\"\"},{\"line\":59,\"text\":\"      ;;\"},{\"line\":60,\"text\":\"    tcp)\"},{\"line\":61,\"text\":\"      timeout 2 bash -c \\\"</dev/tcp/$target/$value\\\" >/dev/null 2>&1\"},{\"line\":62,\"text\":\"      rc=$?\"},{\"line\":63,\"text\":\"      detail=\\\"tcp_${value}_rc=$rc\\\"\"},{\"line\":64,\"text\":\"      ;;\"},{\"line\":65,\"text\":\"    http_status_ok)\"},{\"line\":66,\"text\":\"      line=$(curl -fsS --max-time 5 \\\"$value\\\" 2>/tmp/full-http-$name.err | head -n1)\"},{\"line\":67,\"text\":\"      curlrc=$?\"},{\"line\":68,\"text\":\"      echo \\\"$line\\\" | grep -q '^STATUS='\"},{\"line\":69,\"text\":\"      okrc=$?\"},{\"line\":70,\"text\":\"      [ \\\"$curlrc\\\" = 0 ] && [ \\\"$okrc\\\" = 0 ]\"},{\"line\":71,\"text\":\"      rc=$?\"},{\"line\":72,\"text\":\"      detail=\\\"http_rc=$curlrc ok_rc=$okrc line=$line\\\"\"},{\"line\":73,\"text\":\"      ;;\"},{\"line\":74,\"text\":\"    missing)\"},{\"line\":75,\"text\":\"      rc=1\"},{\"line\":76,\"text\":\"      detail=\\\"missing_target=$target value=$value\\\"\"},{\"line\":77,\"text\":\"      ;;\"},{\"line\":78,\"text\":\"    *)\"},{\"line\":79,\"text\":\"      rc=1\"},{\"line\":80,\"text\":\"      detail=\\\"unknown_kind=$kind\\\"\"},{\"line\":81,\"text\":\"      ;;\"},{\"line\":82,\"text\":\"  esac\"},{\"line\":83,\"text\":\"\"},{\"line\":84,\"text\":\"  if [ \\\"$rc\\\" = 0 ]; then\"},{\"line\":85,\"text\":\"    echo \\\"OK severity=$severity kind=$kind name=$name target=$target value=$value $detail\\\" >> \\\"$TMP\\\"\"},{\"line\":86,\"text\":\"  else\"},{\"line\":87,\"text\":\"    if [ \\\"$severity\\\" = critical ]; then\"},{\"line\":88,\"text\":\"      echo \\\"FAIL severity=critical kind=$kind name=$name target=$target value=$value $detail\\\" >> \\\"$TMP\\\"\"},{\"line\":89,\"text\":\"      CRIT_FAIL=$((CRIT_FAIL+1))\"},{\"line\":90,\"text\":\"      BAD=$((BAD+1))\"},{\"line\":91,\"text\":\"    else\"},{\"line\":92,\"text\":\"      echo \\\"INFO_OFF severity=optional kind=$kind name=$name target=$target value=$value $detail\\\" >> \\\"$TMP\\\"\"},{\"line\":93,\"text\":\"      OPT_OFF=$((OPT_OFF+1))\"},{\"line\":94,\"text\":\"      INFO_OFF=$((INFO_OFF+1))\"},{\"line\":95,\"text\":\"    fi\"},{\"line\":96,\"text\":\"  fi\"},{\"line\":97,\"text\":\"done < \\\"$MAP\\\"\"},{\"line\":98,\"text\":\"\"},{\"line\":99,\"text\":\"SELF_FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":100,\"text\":\"[ \\\"$SELF_FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":101,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":102,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":103,\"text\":\"\"},{\"line\":104,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"},{\"line\":106,\"text\":\"echo \\\"$LINE\\\" | tee \\\"$HEALTH\\\" >/dev/null\"},{\"line\":107,\"text\":\"{\"},{\"line\":108,\"text\":\"  echo\"},{\"line\":109,\"text\":\"  echo \\\"## Result\\\"\"},{\"line\":110,\"text\":\"  echo \\\"$LINE\\\"\"},{\"line\":111,\"text\":\"} >> \\\"$TMP\\\"\"},{\"line\":112,\"text\":\"mv \\\"$TMP\\\" \\\"$REPORT\\\"\"},{\"line\":113,\"text\":\"cat \\\"$HEALTH\\\"\"}]},{\"bytes\":1433,\"line_count\":27,\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"priority\":92,\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set -Eeuo pipefail\"},{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"},{\"line\":6,\"text\":\"ORIGINAL_RC=0\"},{\"line\":7,\"text\":\"\\\"$ORIGINAL\\\" || ORIGINAL_RC=$?\"},{\"line\":8,\"text\":\"[ -s \\\"$HEALTH\\\" ] || exit \\\"$ORIGINAL_RC\\\"\"},{\"line\":9,\"text\":\"line=\\\"$(sed -n \\\"1p\\\" \\\"$HEALTH\\\" 2>/dev/null || true)\\\"\"},{\"line\":10,\"text\":\"field(){ local key=<REDACTED>\"},{\"line\":11,\"text\":\"self_failed=\\\"$(field SELF_FAILED_UNITS || true)\\\"\"},{\"line\":12,\"text\":\"qga_state=\\\"$(field QGA || true)\\\"\"},{\"line\":13,\"text\":\"backup_state=\\\"$(field BACKUP || true)\\\"\"},{\"line\":14,\"text\":\"if [ \\\"$ORIGINAL_RC\\\" -eq 0 ] && [ \\\"$self_failed\\\" = \\\"0\\\" ] && [ \\\"$qga_state\\\" = \\\"active\\\" ] && [ \\\"$backup_state\\\" = \\\"MAIL_CLOUD_2COPY_VERIFY_OK\\\" ]; then\"},{\"line\":15,\"text\":\"  normalized=\\\"$(printf \\\"%s\\\\n\\\" \\\"$line\\\" | sed -E \\\"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\\\")\\\"\"},{\"line\":16,\"text\":\"  printf \\\"%s\\\\n\\\" \\\"$normalized\\\" > \\\"$HEALTH\\\"\"},{\"line\":17,\"text\":\"  if [ -f \\\"$STATUS_FILE\\\" ]; then\"},{\"line\":18,\"text\":\"    status_line=\\\"$(sed -n \\\"1p\\\" \\\"$STATUS_FILE\\\" 2>/dev/null || true)\\\"\"},{\"line\":19,\"text\":\"    status_normalized=\\\"$(printf \\\"%s\\\\n\\\" \\\"$status_line\\\" | sed -E \\\"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\\\")\\\"\"},{\"line\":20,\"text\":\"    tmp=\\\"${STATUS_FILE}.tmp.$$\\\"\"},{\"line\":21,\"text\":\"    { printf \\\"%s\\\\n\\\" \\\"$status_normalized\\\"; tail -n +2 \\\"$STATUS_FILE\\\" 2>/dev/null || true; } > \\\"$tmp\\\"\"},{\"line\":22,\"text\":\"    chown --reference=\\\"$STATUS_FILE\\\" \\\"$tmp\\\" 2>/dev/null || true\"},{\"line\":23,\"text\":\"    chmod --reference=\\\"$STATUS_FILE\\\" \\\"$tmp\\\" 2>/dev/null || true\"},{\"line\":24,\"text\":\"    mv \\\"$tmp\\\" \\\"$STATUS_FILE\\\"\"},{\"line\":25,\"text\":\"  fi\"},{\"line\":26,\"text\":\"fi\"},{\"line\":27,\"text\":\"exit \\\"$ORIGINAL_RC\\\"\"}]},{\"bytes\":848,\"line_count\":17,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"priority\":92,\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"BAD=0\"},{\"line\":5,\"text\":\"HOST=$(hostname 2>/dev/null || echo unknown)\"},{\"line\":6,\"text\":\"IP_OK=0\"},{\"line\":7,\"text\":\"ip -4 -br a | grep -q \\\"[PRIVATE_IP]\\\" && IP_OK=1 || BAD=$((BAD+1))\"},{\"line\":8,\"text\":\"DNS_OK=0\"},{\"line\":9,\"text\":\"getent hosts pve01 >/dev/null 2>&1 || getent hosts gram1.ru >/dev/null 2>&1\"},{\"line\":10,\"text\":\"[ \\\"$?\\\" = 0 ] && DNS_OK=1 || BAD=$((BAD+1))\"},{\"line\":11,\"text\":\"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":12,\"text\":\"[ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":13,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":14,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":15,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}]},{\"bytes\":2541,\"line_count\":105,\"mode\":\"0o750\",\"path\":\"/usr/local/sbin/homelab-stage4i-verify\",\"priority\":92,\"sha256\":\"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set -euo pipefail\"},{\"line\":3,\"text\":\"export LC_ALL=C\"},{\"line\":4,\"text\":\"\"},{\"line\":5,\"text\":\"MARKER=\\\"CR0019_STAGE4I_IMMUTABLE_V1\\\"\"},{\"line\":6,\"text\":\"STATE_DIR=\\\"/var/lib/homelab-cluster-admin/stage4i-verification\\\"\"},{\"line\":7,\"text\":\"REPORT=\\\"$STATE_DIR/latest.env\\\"\"},{\"line\":8,\"text\":\"TMP=\\\"$(mktemp)\\\"\"},{\"line\":9,\"text\":\"\"},{\"line\":10,\"text\":\"cleanup() {\"},{\"line\":11,\"text\":\"    rm -f \\\"$TMP\\\"\"},{\"line\":12,\"text\":\"}\"},{\"line\":13,\"text\":\"trap cleanup EXIT\"},{\"line\":14,\"text\":\"\"},{\"line\":15,\"text\":\"mapfile -t ACTIVATION_FILES < <(\"},{\"line\":16,\"text\":\"    find \\\\\"},{\"line\":17,\"text\":\"        /var/lib/homelab-cluster-admin \\\\\"},{\"line\":18,\"text\":\"        /opt \\\\\"},{\"line\":19,\"text\":\"        /srv \\\\\"},{\"line\":20,\"text\":\"        /usr/local/share \\\\\"},{\"line\":21,\"text\":\"        -type f \\\\\"},{\"line\":22,\"text\":\"        -path '*/.stage4i-control/activation.env' \\\\\"},{\"line\":23,\"text\":\"        -print 2>/dev/null |\"},{\"line\":24,\"text\":\"    sort -u\"},{\"line\":25,\"text\":\")\"},{\"line\":26,\"text\":\"\"},{\"line\":27,\"text\":\"if [ \\\"${#ACTIVATION_FILES[@]}\\\" -ne 1 ]; then\"},{\"line\":28,\"text\":\"    echo \\\"STAGE4I_VERIFICATION_STATUS=FAILED\\\"\"},{\"line\":29,\"text\":\"    echo \\\"REASON=ACTIVATION_FILE_COUNT_${#ACTIVATION_FILES[@]}\\\"\"},{\"line\":30,\"text\":\"    exit 50\"},{\"line\":31,\"text\":\"fi\"},{\"line\":32,\"text\":\"\"},{\"line\":33,\"text\":\"ACTIVATION_FILE=\\\"${ACTIVATION_FILES[0]}\\\"\"},{\"line\":34,\"text\":\"CONTROL_DIR=\\\"$(dirname \\\"$ACTIVATION_FILE\\\")\\\"\"},{\"line\":35,\"text\":\"ACTIVE_LINK=\\\"$CONTROL_DIR/active\\\"\"},{\"line\":36,\"text\":\"ROLLBACK_LINK=\\\"$CONTROL_DIR/rollback\\\"\"},{\"line\":37,\"text\":\"\"},{\"line\":38,\"text\":\"# shellcheck disable=SC1090\"},{\"line\":39,\"text\":\"source \\\"$ACTIVATION_FILE\\\"\"},{\"line\":40,\"text\":\"\"},{\"line\":41,\"text\":\"test \\\"${STAGE4I_ACTIVATION_STATUS:-}\\\" = \\\"ACTIVE\\\"\"},{\"line\":42,\"text\":\"test -L \\\"$ACTIVE_LINK\\\"\"},{\"line\":43,\"text\":\"test -L \\\"$ROLLBACK_LINK\\\"\"},{\"line\":44,\"text\":\"\"},{\"line\":45,\"text\":\"ACTIVE_RESOLVED=\\\"$(readlink -f \\\"$ACTIVE_LINK\\\")\\\"\"},{\"line\":46,\"text\":\"ROLLBACK_RESOLVED=\\\"$(readlink -f \\\"$ROLLBACK_LINK\\\")\\\"\"},{\"line\":47,\"text\":\"\"},{\"line\":48,\"text\":\"test \\\"$ACTIVE_RESOLVED\\\" = \\\"${ACTIVE_PATH:-}\\\"\"},{\"line\":49,\"text\":\"test \\\"$ROLLBACK_RESOLVED\\\" = \\\"${ROLLBACK_PATH:-}\\\"\"},{\"line\":50,\"text\":\"test -d \\\"$ACTIVE_RESOLVED\\\"\"},{\"line\":51,\"text\":\"test -d \\\"$ROLLBACK_RESOLVED\\\"\"},{\"line\":52,\"text\":\"\"},{\"line\":53,\"text\":\"META=\\\"$ACTIVE_RESOLVED/IMMUTABLE.env\\\"\"},{\"line\":54,\"text\":\"MANIFEST=\\\"$ACTIVE_RESOLVED/MANIFEST.sha256\\\"\"},{\"line\":55,\"text\":\"\"},{\"line\":56,\"text\":\"test -f \\\"$META\\\"\"},{\"line\":57,\"text\":\"test -f \\\"$MANIFEST\\\"\"},{\"line\":58,\"text\":\"grep -q \\\"^BUILD_MARKER=$MARKER$\\\" \\\"$META\\\"\"},{\"line\":59,\"text\":\"grep -q '^TASK_STAGE=4I$' \\\"$META\\\"\"},{\"line\":60,\"text\":\"grep -q '^TASK_STATUS=IMMUTABLE$' \\\"$META\\\"\"},{\"line\":61,\"text\":\"\"},{\"line\":62,\"text\":\"(\"},{\"line\":63,\"text\":\"    cd \\\"$ACTIVE_RESOLVED\\\"\"},{\"line\":64,\"text\":\"    sha256sum -c MANIFEST.sha256 >/dev/null\"},{\"line\":65,\"text\":\")\"},{\"line\":66,\"text\":\"\"},{\"line\":67,\"text\":\"if find \\\"$ACTIVE_RESOLVED\\\" \\\\\"},{\"line\":68,\"text\":\"    \\\\( ! -user root -o ! -group root \\\\) \\\\\"},{\"line\":69,\"text\":\"    -print -quit |\"},{\"line\":70,\"text\":\"    grep -q .\"},{\"line\":71,\"text\":\"then\"},{\"line\":72,\"text\":\"    echo \\\"STAGE4I_VERIFICATION_STATUS=FAILED\\\"\"},{\"line\":73,\"text\":\"    echo \\\"REASON=INVALID_OWNER\\\"\"},{\"line\":74,\"text\":\"    exit 51\"},{\"line\":75,\"text\":\"fi\"},{\"line\":76,\"text\":\"\"},{\"line\":77,\"text\":\"if find \\\"$ACTIVE_RESOLVED\\\" \\\\\"},{\"line\":78,\"text\":\"    -perm /0222 \\\\\"},{\"line\":79,\"text\":\"    -print -quit |\"},{\"line\":80,\"text\":\"    grep -q .\"},{\"line\":81,\"text\":\"then\"},{\"line\":82,\"text\":\"    echo \\\"STAGE4I_VERIFICATION_STATUS=FAILED\\\"\"},{\"line\":83,\"text\":\"    echo \\\"REASON=WRITABLE_IMMUTABLE_CONTENT\\\"\"},{\"line\":84,\"text\":\"    exit 52\"},{\"line\":85,\"text\":\"fi\"},{\"line\":86,\"text\":\"\"},{\"line\":87,\"text\":\"install -d -m 0750 -o root -g root \\\"$STATE_DIR\\\"\"},{\"line\":88,\"text\":\"\"},{\"line\":89,\"text\":\"{\"},{\"line\":90,\"text\":\"    echo \\\"STAGE4I_VERIFICATION_STATUS=SUCCESS\\\"\"},{\"line\":91,\"text\":\"    echo \\\"VERIFIED_AT_UTC=$(date -u +%Y-%m-%dT%H:%M:%SZ)\\\"\"},{\"line\":92,\"text\":\"    echo \\\"ACTIVE_VERSION=${ACTIVE_VERSION:-UNKNOWN}\\\"\"},{\"line\":93,\"text\":\"    echo \\\"ACTIVE_PATH=$ACTIVE_RESOLVED\\\"\"},{\"line\":94,\"text\":\"    echo \\\"ROLLBACK_VERSION=${ROLLBACK_VERSION:-UNKNOWN}\\\"\"},{\"line\":95,\"text\":\"    echo \\\"ROLLBACK_PATH=$ROLLBACK_RESOLVED\\\"\"},{\"line\":96,\"text\":\"    echo \\\"MANIFEST_SHA256=$(sha256sum \\\"$MANIFEST\\\" | awk '{print $1}')\\\"\"},{\"line\":97,\"text\":\"    echo \\\"VERIFIED_ON_HOST=$(hostname -f 2>/dev/null || hostname)\\\"\"},{\"line\":98,\"text\":\"} >\\\"$TMP\\\"\"},{\"line\":99,\"text\":\"\"},{\"line\":100,\"text\":\"chown root:root \\\"$TMP\\\"\"},{\"line\":101,\"text\":\"chmod 0440 \\\"$TMP\\\"\"},{\"line\":102,\"text\":\"mv -f \\\"$TMP\\\" \\\"$REPORT\\\"\"},{\"line\":103,\"text\":\"trap - EXIT\"},{\"line\":104,\"text\":\"\"},{\"line\":105,\"text\":\"cat \\\"$REPORT\\\"\"}]},{\"bytes\":616,\"line_count\":19,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"priority\":90,\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"Requires=postgresql.service\"},{\"line\":6,\"text\":\"\"},{\"line\":7,\"text\":\"[Service]\"},{\"line\":8,\"text\":\"Type=oneshot\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":13,\"text\":\"UMask=0027\"},{\"line\":14,\"text\":\"NoNewPrivileges=true\"},{\"line\":15,\"text\":\"PrivateTmp=true\"},{\"line\":16,\"text\":\"ProtectSystem=strict\"},{\"line\":17,\"text\":\"ProtectHome=true\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}]},{\"bytes\":231,\"line_count\":12,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"priority\":90,\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=30s\"},{\"line\":6,\"text\":\"OnUnitActiveSec=60s\"},{\"line\":7,\"text\":\"AccuracySec=10s\"},{\"line\":8,\"text\":\"Persistent=true\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"},{\"line\":10,\"text\":\"\"},{\"line\":11,\"text\":\"[Install]\"},{\"line\":12,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":803,\"line_count\":28,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"priority\":90,\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"Requires=postgresql.service\"},{\"line\":6,\"text\":\"\"},{\"line\":7,\"text\":\"[Service]\"},{\"line\":8,\"text\":\"Type=simple\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":13,\"text\":\"ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\"},{\"line\":14,\"text\":\"Restart=on-failure\"},{\"line\":15,\"text\":\"RestartSec=5\"},{\"line\":16,\"text\":\"UMask=0027\"},{\"line\":17,\"text\":\"NoNewPrivileges=true\"},{\"line\":18,\"text\":\"PrivateTmp=true\"},{\"line\":19,\"text\":\"ProtectSystem=strict\"},{\"line\":20,\"text\":\"ProtectHome=true\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"},{\"line\":23,\"text\":\"RestrictSUIDSGID=true\"},{\"line\":24,\"text\":\"LockPersonality=true\"},{\"line\":25,\"text\":\"RestrictRealtime=true\"},{\"line\":26,\"text\":\"\"},{\"line\":27,\"text\":\"[Install]\"},{\"line\":28,\"text\":\"WantedBy=multi-user.target\"}]},{\"bytes\":1190,\"line_count\":42,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"priority\":90,\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":3,\"text\":\"After=network-online.target\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"\"},{\"line\":6,\"text\":\"[Service]\"},{\"line\":7,\"text\":\"Type=oneshot\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":12,\"text\":\"UMask=0077\"},{\"line\":13,\"text\":\"NoNewPrivileges=yes\"},{\"line\":14,\"text\":\"PrivateTmp=yes\"},{\"line\":15,\"text\":\"PrivateDevices=yes\"},{\"line\":16,\"text\":\"ProtectSystem=strict\"},{\"line\":17,\"text\":\"ProtectHome=yes\"},{\"line\":18,\"text\":\"ProtectKernelTunables=yes\"},{\"line\":19,\"text\":\"ProtectKernelModules=yes\"},{\"line\":20,\"text\":\"ProtectKernelLogs=yes\"},{\"line\":21,\"text\":\"ProtectControlGroups=yes\"},{\"line\":22,\"text\":\"ProtectClock=yes\"},{\"line\":23,\"text\":\"ProtectHostname=yes\"},{\"line\":24,\"text\":\"LockPersonality=yes\"},{\"line\":25,\"text\":\"MemoryDenyWriteExecute=yes\"},{\"line\":26,\"text\":\"RestrictSUIDSGID=yes\"},{\"line\":27,\"text\":\"RestrictRealtime=yes\"},{\"line\":28,\"text\":\"RemoveIPC=yes\"},{\"line\":29,\"text\":\"CapabilityBoundingSet=\"},{\"line\":30,\"text\":\"AmbientCapabilities=\"},{\"line\":31,\"text\":\"RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\"},{\"line\":32,\"text\":\"IPAddressDeny=any\"},{\"line\":33,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":34,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":35,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":36,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":40,\"text\":\"StateDirectoryMode=0750\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"},{\"line\":42,\"text\":\"TimeoutStartSec=70\"}]},{\"bytes\":223,\"line_count\":12,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"priority\":90,\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=2min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=60s\"},{\"line\":7,\"text\":\"RandomizedDelaySec=5s\"},{\"line\":8,\"text\":\"Persistent=true\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"},{\"line\":10,\"text\":\"\"},{\"line\":11,\"text\":\"[Install]\"},{\"line\":12,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":133,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"priority\":90,\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}]},{\"bytes\":154,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"priority\":90,\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=5min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=5min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":131,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"priority\":90,\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}]},{\"bytes\":149,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"priority\":90,\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=3min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=5min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":127,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"priority\":90,\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}]},{\"bytes\":151,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"priority\":90,\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=2min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=5min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":138,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"priority\":90,\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}]},{\"bytes\":154,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"priority\":90,\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=1min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=1min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":301,\"line_count\":13,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"priority\":90,\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":3,\"text\":\"After=network-online.target\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"\"},{\"line\":6,\"text\":\"[Service]\"},{\"line\":7,\"text\":\"Type=simple\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"},{\"line\":9,\"text\":\"Restart=always\"},{\"line\":10,\"text\":\"RestartSec=5\"},{\"line\":11,\"text\":\"\"},{\"line\":12,\"text\":\"[Install]\"},{\"line\":13,\"text\":\"WantedBy=multi-user.target\"}]},{\"bytes\":465,\"line_count\":22,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"priority\":90,\"sha256\":\"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Verify active immutable Stage 4I task version\"},{\"line\":3,\"text\":\"After=local-fs.target\"},{\"line\":4,\"text\":\"\"},{\"line\":5,\"text\":\"[Service]\"},{\"line\":6,\"text\":\"Type=oneshot\"},{\"line\":7,\"text\":\"User=root\"},{\"line\":8,\"text\":\"Group=root\"},{\"line\":9,\"text\":\"ExecStart=/usr/local/sbin/homelab-stage4i-verify\"},{\"line\":10,\"text\":\"NoNewPrivileges=true\"},{\"line\":11,\"text\":\"PrivateDevices=true\"},{\"line\":12,\"text\":\"ProtectClock=true\"},{\"line\":13,\"text\":\"ProtectControlGroups=true\"},{\"line\":14,\"text\":\"ProtectHome=true\"},{\"line\":15,\"text\":\"ProtectKernelLogs=true\"},{\"line\":16,\"text\":\"ProtectKernelModules=true\"},{\"line\":17,\"text\":\"ProtectKernelTunables=true\"},{\"line\":18,\"text\":\"RestrictNamespaces=true\"},{\"line\":19,\"text\":\"RestrictRealtime=true\"},{\"line\":20,\"text\":\"LockPersonality=true\"},{\"line\":21,\"text\":\"MemoryDenyWriteExecute=true\"},{\"line\":22,\"text\":\"UMask=0027\"}]},{\"bytes\":224,\"line_count\":12,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"priority\":90,\"sha256\":\"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Regularly verify active immutable Stage 4I task version\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=2min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=15min\"},{\"line\":7,\"text\":\"AccuracySec=1min\"},{\"line\":8,\"text\":\"Persistent=true\"},{\"line\":9,\"text\":\"Unit=homelab-stage4i-verify.service\"},{\"line\":10,\"text\":\"\"},{\"line\":11,\"text\":\"[Install]\"},{\"line\":12,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":380,\"line_count\":1,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"priority\":70,\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\",\"source\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}]},{\"bytes\":217,\"line_count\":1,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"priority\":70,\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\",\"source\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}]},{\"bytes\":128,\"line_count\":1,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"priority\":70,\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\",\"source\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md\"}]},{\"bytes\":1535,\"line_count\":1,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"priority\":70,\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\",\"source\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}]},{\"bytes\":89,\"line_count\":3,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"priority\":50,\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\",\"source\":[{\"line\":1,\"text\":\"[Service]\"},{\"line\":2,\"text\":\"ExecStart=\"},{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}]},{\"bytes\":28599,\"line_count\":484,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"priority\":50,\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\",\"source\":[{\"line\":1,\"text\":\"<REDACTED_SECRET_LINE>\"},{\"line\":2,\"text\":\"from collections import defaultdict\"},{\"line\":3,\"text\":\"from contextlib import contextmanager\"},{\"line\":4,\"text\":\"from datetime import datetime\"},{\"line\":5,\"text\":\"from urllib.parse import parse_qs\"},{\"line\":6,\"text\":\"import psycopg2\"},{\"line\":7,\"text\":\"from psycopg2.extras import RealDictCursor\"},{\"line\":8,\"text\":\"from fastapi import FastAPI, HTTPException, Request\"},{\"line\":9,\"text\":\"from fastapi.responses import HTMLResponse, RedirectResponse\"},{\"line\":10,\"text\":\"from starlette.middleware.trustedhost import TrustedHostMiddleware\"},{\"line\":11,\"text\":\"\"},{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =<REDACTED>\"},{\"line\":14,\"text\":\"COOKIE =<REDACTED>\"},{\"line\":15,\"text\":\"failures, locks = defaultdict(list), {}\"},{\"line\":16,\"text\":\"app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":18,\"text\":\"\"},{\"line\":19,\"text\":\"def db():\"},{\"line\":20,\"text\":\"    return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":21,\"text\":\"\"},{\"line\":22,\"text\":\"def enc(value):\"},{\"line\":23,\"text\":\"    return base64.urlsafe_b64encode(value).rstrip(b\\\"=\\\").decode()\"},{\"line\":24,\"text\":\"\"},{\"line\":25,\"text\":\"def dec(value):\"},{\"line\":26,\"text\":\"    return base64.urlsafe_b64decode(value + \\\"=\\\" * (-len(value) % 4))\"},{\"line\":27,\"text\":\"\"},{\"line\":28,\"text\":\"def sign(value):\"},{\"line\":29,\"text\":\"    return enc(hmac.new(KEY, value.encode(), hashlib.sha256).digest())\"},{\"line\":30,\"text\":\"\"},{\"line\":31,\"text\":\"def ip(request):\"},{\"line\":32,\"text\":\"    peer = request.client.host if request.client else \\\"unknown\\\"\"},{\"line\":33,\"text\":\"    if peer in {\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"::1\\\"}:\"},{\"line\":34,\"text\":\"        return request.headers.get(\\\"x-forwarded-for\\\", peer).split(\\\",\\\")[0].strip()\"},{\"line\":35,\"text\":\"    return peer\"},{\"line\":36,\"text\":\"\"},{\"line\":37,\"text\":\"def login_token(request): <REDACTED>\"},{\"line\":38,\"text\":\"    raw = f\\\"{int(time.time())}:{ip(request)}\\\"\"},{\"line\":39,\"text\":\"    return raw + \\\".\\\" + sign(raw)\"},{\"line\":40,\"text\":\"\"},{\"line\":41,\"text\":\"def valid_login_token(request, token): <REDACTED>\"},{\"line\":42,\"text\":\"    try:\"},{\"line\":43,\"text\":\"        raw, signature =<REDACTED>\"},{\"line\":44,\"text\":\"        stamp, address = raw.split(\\\":\\\", 1)\"},{\"line\":45,\"text\":\"        return hmac.compare_digest(signature, sign(raw)) and address == ip(request) and 0 <= time.time() - int(stamp) <= 600\"},{\"line\":46,\"text\":\"    except Exception:\"},{\"line\":47,\"text\":\"        return False\"},{\"line\":48,\"text\":\"\"},{\"line\":49,\"text\":\"def session_cookie(request): <REDACTED>\"},{\"line\":50,\"text\":\"    data =<REDACTED>\"},{\"line\":51,\"text\":\"    raw = enc(json.dumps(data,separators=(\\\",\\\",\\\":\\\")).encode())\"},{\"line\":52,\"text\":\"    return raw + \\\".\\\" + sign(raw)\"},{\"line\":53,\"text\":\"\"},{\"line\":54,\"text\":\"def session(request):\"},{\"line\":55,\"text\":\"    try:\"},{\"line\":56,\"text\":\"        raw, signature =<REDACTED>\"},{\"line\":57,\"text\":\"        data = json.loads(dec(raw))\"},{\"line\":58,\"text\":\"        expected = hashlib.sha256(request.headers.get(\\\"user-agent\\\",\\\"\\\").encode()).hexdigest()[:24]\"},{\"line\":59,\"text\":\"        return data if hmac.compare_digest(signature,sign(raw)) and data[\\\"exp\\\"] >= time.time() and data[\\\"ua\\\"] == expected else None\"},{\"line\":60,\"text\":\"    except Exception:\"},{\"line\":61,\"text\":\"        return None\"},{\"line\":62,\"text\":\"\"},{\"line\":63,\"text\":\"def password_ok(password): <REDACTED>\"},{\"line\":64,\"text\":\"    try:\"},{\"line\":65,\"text\":\"        scheme, rounds, salt, expected = open(PASSFILE,encoding=\\\"utf-8\\\").read().strip().split(\\\"$\\\",3)\"},{\"line\":66,\"text\":\"        actual =<REDACTED>\"},{\"line\":67,\"text\":\"        return scheme == \\\"pbkdf2_sha256\\\" and hmac.compare_digest(base64.urlsafe_b64encode(actual).decode(),expected)\"},{\"line\":68,\"text\":\"    except Exception:\"},{\"line\":69,\"text\":\"        return False\"},{\"line\":70,\"text\":\"\"},{\"line\":71,\"text\":\"def login_page(request, error=\\\"\\\"):\"},{\"line\":72,\"text\":\"    token =<REDACTED>\"},{\"line\":73,\"text\":\"    return f\\\"\\\"\\\"<!doctype html><html lang=<REDACTED>\"},{\"line\":74,\"text\":\"\"},{\"line\":75,\"text\":\"def status_kind(value):\"},{\"line\":76,\"text\":\"    text = \\\"UNKNOWN\\\" if value is None or str(value).strip() == \\\"\\\" else str(value).strip().upper()\"},{\"line\":77,\"text\":\"    if text in {\\\"OK\\\", \\\"ACTIVE\\\", \\\"RUNNING\\\", \\\"UP\\\", \\\"ONLINE\\\", \\\"HEALTHY\\\", \\\"RECOVERED\\\", \\\"CLOSED\\\", \\\"SUCCESS\\\", \\\"ENABLED\\\", \\\"READY\\\"}:\"},{\"line\":78,\"text\":\"        return \\\"ok\\\"\"},{\"line\":79,\"text\":\"    if text in {\\\"WARN\\\", \\\"WARNING\\\", \\\"PENDING\\\", \\\"DEGRADED\\\", \\\"ATTENTION\\\", \\\"STALE\\\"}:\"},{\"line\":80,\"text\":\"        return \\\"warn\\\"\"},{\"line\":81,\"text\":\"    if text in {\\\"OPEN\\\", \\\"ERROR\\\", \\\"FAILED\\\", \\\"DOWN\\\", \\\"OFFLINE\\\", \\\"CRITICAL\\\", \\\"P0\\\", \\\"P1\\\", \\\"FIRING\\\", \\\"UNHEALTHY\\\"}:\"},{\"line\":82,\"text\":\"        return \\\"bad\\\"\"},{\"line\":83,\"text\":\"    return \\\"unknown\\\"\"},{\"line\":84,\"text\":\"\"},{\"line\":85,\"text\":\"def status_badge(value):\"},{\"line\":86,\"text\":\"    text = \\\"UNKNOWN\\\" if value is None or str(value).strip() == \\\"\\\" else str(value)\"},{\"line\":87,\"text\":\"    kind = status_kind(text)\"},{\"line\":88,\"text\":\"    return \\\"<span class=\\\\\\\"state state-\\\" + kind + \\\"\\\\\\\"><span class=\\\\\\\"state-dot\\\\\\\"></span>\\\" + html.escape(text) + \\\"</span>\\\"\"},{\"line\":89,\"text\":\"\"},{\"line\":90,\"text\":\"def summary_card(title, value, state):\"},{\"line\":91,\"text\":\"    return \\\"<div class=\\\\\\\"card\\\\\\\"><div class=\\\\\\\"card-health\\\\\\\">\\\" + status_badge(state) + \\\"</div><b>\\\" + html.escape(str(value)) + \\\"</b><span>\\\" + html.escape(title) + \\\"</span></div>\\\"\"},{\"line\":92,\"text\":\"\"},{\"line\":93,\"text\":\"def aggregate_state(rows, keys=(\\\"status\\\",), empty_state=\\\"WARN\\\"):\"},{\"line\":94,\"text\":\"    if not rows:\"},{\"line\":95,\"text\":\"        return empty_state\"},{\"line\":96,\"text\":\"    kinds = []\"},{\"line\":97,\"text\":\"    for row in rows:\"},{\"line\":98,\"text\":\"        for key in keys:\"},{\"line\":99,\"text\":\"            if row.get(key) is not None:\"},{\"line\":100,\"text\":\"                kinds.append(status_kind(row.get(key)))\"},{\"line\":101,\"text\":\"    if \\\"bad\\\" in kinds:\"},{\"line\":102,\"text\":\"        return \\\"ERROR\\\"\"},{\"line\":103,\"text\":\"    if \\\"warn\\\" in kinds or \\\"unknown\\\" in kinds or not kinds:\"},{\"line\":104,\"text\":\"        return \\\"WARN\\\"\"},{\"line\":105,\"text\":\"    return \\\"OK\\\"\"},{\"line\":106,\"text\":\"\"},{\"line\":107,\"text\":\"def incident_section_state(rows):\"},{\"line\":108,\"text\":\"    if not rows:\"},{\"line\":109,\"text\":\"        return \\\"OK\\\"\"},{\"line\":110,\"text\":\"    severe = {\\\"P0\\\", \\\"P1\\\", \\\"CRITICAL\\\", \\\"ERROR\\\"}\"},{\"line\":111,\"text\":\"    for row in rows:\"},{\"line\":112,\"text\":\"        severity = str(row.get(\\\"severity\\\") or \\\"\\\").strip().upper()\"},{\"line\":113,\"text\":\"        state = row.get(\\\"status\\\")\"},{\"line\":114,\"text\":\"        if severity in severe or status_kind(state) == \\\"bad\\\":\"},{\"line\":115,\"text\":\"            return \\\"ERROR\\\"\"},{\"line\":116,\"text\":\"    return \\\"WARN\\\"\"},{\"line\":117,\"text\":\"\"},{\"line\":118,\"text\":\"# Все текущие и будущие подробные блоки dashboard должны использовать этот компонент.\"},{\"line\":119,\"text\":\"def health_section(title, count, state, content, wide=False):\"},{\"line\":120,\"text\":\"    kind = status_kind(state)\"},{\"line\":121,\"text\":\"    css_class = \\\"panel health-section\\\" + (\\\" wide\\\" if wide else \\\"\\\")\"},{\"line\":122,\"text\":\"    return \\\"<details class=\\\\\\\"\\\" + css_class + \\\"\\\\\\\"><summary><span class=\\\\\\\"section-heading\\\\\\\"><span class=\\\\\\\"state state-\\\" + kind + \\\"\\\\\\\" aria-label=\\\\\\\"\\\" + html.escape(str(state)) + \\\"\\\\\\\"><span class=\\\\\\\"state-dot\\\\\\\"></span></span><span class=\\\\\\\"section-title\\\\\\\">\\\" + html.escape(str(title)) + \\\"</span><span class=\\\\\\\"section-count\\\\\\\">\\\" + html.escape(str(count)) + \\\"</span></span><span class=\\\\\\\"section-chevron\\\\\\\" aria-hidden=\\\\\\\"true\\\\\\\">⌄</span></summary><div class=\\\\\\\"section-body\\\\\\\">\\\" + content + \\\"</div></details>\\\"\"},{\"line\":123,\"text\":\"\"},{\"line\":124,\"text\":\"def table(rows, columns):\"},{\"line\":125,\"text\":\"    if not rows:\"},{\"line\":126,\"text\":\"        return \\\"<p class=\\\\\\\"muted\\\\\\\">Данные ещё не собраны.</p>\\\"\"},{\"line\":127,\"text\":\"    head = \\\"\\\".join(\\\"<th>\\\" + html.escape(str(title)) + \\\"</th>\\\" for title, key in columns)\"},{\"line\":128,\"text\":\"    body = []\"},{\"line\":129,\"text\":\"    for row in rows:\"},{\"line\":130,\"text\":\"        cells = []\"},{\"line\":131,\"text\":\"        for title, key in columns:\"},{\"line\":132,\"text\":\"            value = row.get(key)\"},{\"line\":133,\"text\":\"            rendered = status_badge(value) if key in {\\\"status\\\", \\\"severity\\\"} else html.escape(str(value if value is not None else chr(8212)))\"},{\"line\":134,\"text\":\"            cells.append(\\\"<td>\\\" + rendered + \\\"</td>\\\")\"},{\"line\":135,\"text\":\"        row_kind = status_kind(row.get(\\\"status\\\", row.get(\\\"severity\\\", \\\"UNKNOWN\\\")))\"},{\"line\":136,\"text\":\"        body.append(\\\"<tr class=\\\\\\\"row-\\\" + row_kind + \\\"\\\\\\\">\\\" + \\\"\\\".join(cells) + \\\"</tr>\\\")\"},{\"line\":137,\"text\":\"    return \\\"<table><thead><tr>\\\" + head + \\\"</tr></thead><tbody>\\\" + \\\"\\\".join(body) + \\\"</tbody></table>\\\"\"},{\"line\":138,\"text\":\"\"},{\"line\":139,\"text\":\"@app.middleware(\\\"http\\\")\"},{\"line\":140,\"text\":\"async def headers(request, call_next):\"},{\"line\":141,\"text\":\"    response = await call_next(request)\"},{\"line\":142,\"text\":\"    q = chr(39)\"},{\"line\":143,\"text\":\"    response.headers[\\\"Content-Security-Policy\\\"] = f\\\"default-src {q}self{q};style-src {q}self{q} {q}unsafe-inline{q};frame-ancestors {q}none{q};base-uri {q}none{q};form-action {q}self{q}\\\"\"},{\"line\":144,\"text\":\"    response.headers[\\\"X-Frame-Options\\\"] = \\\"DENY\\\"\"},{\"line\":145,\"text\":\"    response.headers[\\\"X-Content-Type-Options\\\"] = \\\"nosniff\\\"\"},{\"line\":146,\"text\":\"    response.headers[\\\"Referrer-Policy\\\"] = \\\"no-referrer\\\"\"},{\"line\":147,\"text\":\"    response.headers[\\\"Cache-Control\\\"] = \\\"no-store\\\"\"},{\"line\":148,\"text\":\"    return response\"},{\"line\":149,\"text\":\"\"},{\"line\":150,\"text\":\"@app.get(\\\"/health\\\")\"},{\"line\":151,\"text\":\"def health():\"},{\"line\":152,\"text\":\"    return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":153,\"text\":\"\"},{\"line\":154,\"text\":\"@app.get(\\\"/login\\\",response_class=HTMLResponse)\"},{\"line\":155,\"text\":\"def get_login(request:Request):\"},{\"line\":156,\"text\":\"    return RedirectResponse(\\\"/\\\",303) if session(request) else HTMLResponse(login_page(request))\"},{\"line\":157,\"text\":\"\"},{\"line\":158,\"text\":\"@app.post(\\\"/login\\\",response_class=HTMLResponse)\"},{\"line\":159,\"text\":\"async def post_login(request:Request):\"},{\"line\":160,\"text\":\"    address, now = ip(request), time.time()\"},{\"line\":161,\"text\":\"    failures[address] = [stamp for stamp in failures[address] if now-stamp < 900]\"},{\"line\":162,\"text\":\"    if locks.get(address,0) > now:\"},{\"line\":163,\"text\":\"        return HTMLResponse(login_page(request,\\\"Слишком много попыток. Повторите позже.\\\"),429)\"},{\"line\":164,\"text\":\"    if int(request.headers.get(\\\"content-length\\\",\\\"0\\\") or 0) > 4096:\"},{\"line\":165,\"text\":\"        raise HTTPException(413)\"},{\"line\":166,\"text\":\"    form = parse_qs((await request.body()).decode(\\\"utf-8\\\",\\\"replace\\\"))\"},{\"line\":167,\"text\":\"    if not valid_login_token(request,form.get(\\\"csrf\\\",[\\\"\\\"])[0]) or not password_ok(form.get(\\\"password\\\",[\\\"\\\"])[0]): <REDACTED>\"},{\"line\":168,\"text\":\"        failures[address].append(now)\"},{\"line\":169,\"text\":\"        if len(failures[address]) >= 5:\"},{\"line\":170,\"text\":\"            locks[address], failures[address] = now+900, []\"},{\"line\":171,\"text\":\"        return HTMLResponse(login_page(request,\\\"Неверный пароль.\\\"),401)\"},{\"line\":172,\"text\":\"    failures.pop(address,None); locks.pop(address,None)\"},{\"line\":173,\"text\":\"    response = RedirectResponse(\\\"/\\\",303)\"},{\"line\":174,\"text\":\"    response.set_cookie(COOKIE,session_cookie(request),max_age=<REDACTED>\"},{\"line\":175,\"text\":\"    return response\"},{\"line\":176,\"text\":\"\"},{\"line\":177,\"text\":\"@app.post(\\\"/logout\\\")\"},{\"line\":178,\"text\":\"async def logout(request:Request):\"},{\"line\":179,\"text\":\"    data = session(request)\"},{\"line\":180,\"text\":\"    if not data:\"},{\"line\":181,\"text\":\"        raise HTTPException(401)\"},{\"line\":182,\"text\":\"    form = parse_qs((await request.body()).decode(\\\"utf-8\\\",\\\"replace\\\"))\"},{\"line\":183,\"text\":\"    if not hmac.compare_digest(form.get(\\\"csrf\\\",[\\\"\\\"])[0],data[\\\"csrf\\\"]):\"},{\"line\":184,\"text\":\"        raise HTTPException(403)\"},{\"line\":185,\"text\":\"    response = RedirectResponse(\\\"/login\\\",303)\"},{\"line\":186,\"text\":\"    response.delete_cookie(COOKIE,path=<REDACTED>\"},{\"line\":187,\"text\":\"    return response\"},{\"line\":188,\"text\":\"\"},{\"line\":189,\"text\":\"@app.get(\\\"/api/summary\\\")\"},{\"line\":190,\"text\":\"def api_summary(request:Request):\"},{\"line\":191,\"text\":\"    if not session(request):\"},{\"line\":192,\"text\":\"        raise HTTPException(401)\"},{\"line\":193,\"text\":\"    with db() as connection, connection.cursor(cursor_factory=RealDictCursor) as cursor:\"},{\"line\":194,\"text\":\"        cursor.execute(\\\"SELECT (SELECT count(*) FROM nodes) nodes,(SELECT count(*) FROM guests) guests,(SELECT count(*) FROM services) services,(SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)) incidents\\\",(\\\"RECOVERED\\\",\\\"CLOSED\\\"))\"},{\"line\":195,\"text\":\"        return dict(cursor.fetchone())\"},{\"line\":196,\"text\":\"\"},{\"line\":197,\"text\":\"@app.get(\\\"/\\\",response_class=HTMLResponse)\"},{\"line\":198,\"text\":\"def dashboard(request:Request):\"},{\"line\":199,\"text\":\"    data = session(request)\"},{\"line\":200,\"text\":\"    if not data:\"},{\"line\":201,\"text\":\"        return RedirectResponse(\\\"/login\\\",303)\"},{\"line\":202,\"text\":\"    with db() as connection, connection.cursor(cursor_factory=RealDictCursor) as cursor:\"},{\"line\":203,\"text\":\"        cursor.execute(\\\"SELECT (SELECT count(*) FROM nodes) nodes,(SELECT count(*) FROM guests) guests,(SELECT count(*) FROM services) services,(SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)) incidents\\\",(\\\"RECOVERED\\\",\\\"CLOSED\\\"))\"},{\"line\":204,\"text\":\"        counts = dict(cursor.fetchone())\"},{\"line\":205,\"text\":\"        cursor.execute(\\\"SELECT name,role,status,last_seen FROM nodes ORDER BY name\\\"); nodes=[dict(x) for x in cursor.fetchall()]\"},{\"line\":206,\"text\":\"        cursor.execute(\\\"SELECT name,scope,host_name,container_name,status,last_seen FROM services ORDER BY name LIMIT 250\\\"); services=[dict(x) for x in cursor.fetchall()]\"},{\"line\":207,\"text\":\"        cursor.execute(\\\"SELECT severity,title,status,last_seen FROM incidents WHERE status NOT IN (%s,%s) ORDER BY last_seen DESC LIMIT 100\\\",(\\\"RECOVERED\\\",\\\"CLOSED\\\")); incidents=[dict(x) for x in cursor.fetchall()]\"},{\"line\":208,\"text\":\"    cards = \\\"\\\".join([summary_card(\\\"Ноды\\\", counts[\\\"nodes\\\"], \\\"OK\\\" if counts[\\\"nodes\\\"] > 0 else \\\"ERROR\\\"), summary_card(\\\"VM и CT\\\", counts[\\\"guests\\\"], \\\"OK\\\" if counts[\\\"guests\\\"] > 0 else \\\"ERROR\\\"), summary_card(\\\"Сервисы\\\", counts[\\\"services\\\"], \\\"OK\\\" if counts[\\\"services\\\"] > 0 else \\\"ERROR\\\"), summary_card(\\\"Инциденты\\\", counts[\\\"incidents\\\"], \\\"ERROR\\\" if counts[\\\"incidents\\\"] > 0 else \\\"OK\\\")])\"},{\"line\":209,\"text\":\"    csrf = data[\\\"csrf\\\"]\"},{\"line\":210,\"text\":\"    nodes_table = table(nodes,[(\\\"Нода\\\",\\\"name\\\"),(\\\"Роль\\\",\\\"role\\\"),(\\\"Статус\\\",\\\"status\\\"),(\\\"Последняя связь\\\",\\\"last_seen\\\")])\"},{\"line\":211,\"text\":\"    incidents_table = table(incidents,[(\\\"Severity\\\",\\\"severity\\\"),(\\\"Инцидент\\\",\\\"title\\\"),(\\\"Статус\\\",\\\"status\\\"),(\\\"Обновлён\\\",\\\"last_seen\\\")])\"},{\"line\":212,\"text\":\"    services_table = table(services,[(\\\"Сервис\\\",\\\"name\\\"),(\\\"Scope\\\",\\\"scope\\\"),(\\\"Хост\\\",\\\"host_name\\\"),(\\\"Контейнер\\\",\\\"container_name\\\"),(\\\"Статус\\\",\\\"status\\\"),(\\\"Последняя связь\\\",\\\"last_seen\\\")])\"},{\"line\":213,\"text\":\"    nodes_state = aggregate_state(nodes, (\\\"status\\\",), \\\"ERROR\\\")\"},{\"line\":214,\"text\":\"    incidents_state = incident_section_state(incidents)\"},{\"line\":215,\"text\":\"    services_state = aggregate_state(services, (\\\"status\\\",), \\\"ERROR\\\")\"},{\"line\":216,\"text\":\"    nodes_html = health_section(\\\"Ноды\\\", len(nodes), nodes_state, nodes_table)\"},{\"line\":217,\"text\":\"    incidents_html = health_section(\\\"Активные инциденты\\\", len(incidents), incidents_state, incidents_table)\"},{\"line\":218,\"text\":\"    services_html = health_section(\\\"Сервисы\\\", len(services), services_state, services_table, wide=True)\"},{\"line\":219,\"text\":\"    engine_badge = status_badge(\\\"OK\\\")\"},{\"line\":220,\"text\":\"    page = f\\\"\\\"\\\"<!doctype html><html lang=\\\"ru\\\"><head><meta charset=\\\"utf-8\\\"><meta name=\\\"viewport\\\" content=\\\"width=device-width,initial-scale=1\\\"><meta http-equiv=\\\"refresh\\\" content=\\\"30\\\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\\\"post\\\" action=\\\"/logout\\\"><input type=\\\"hidden\\\" name=\\\"csrf\\\" value=\\\"{csrf}\\\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\\\"\\\"\\\"\"},{\"line\":221,\"text\":\"    return HTMLResponse(page)\"},{\"line\":222,\"text\":\"\"},{\"line\":223,\"text\":\"from fastapi.responses import PlainTextResponse\"},{\"line\":224,\"text\":\"\"},{\"line\":225,\"text\":\"@app.get(\\\"/metrics\\\", response_class=PlainTextResponse)\"},{\"line\":226,\"text\":\"def metrics():\"},{\"line\":227,\"text\":\"    with db() as connection, connection.cursor() as cursor:\"},{\"line\":228,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM nodes\\\")\"},{\"line\":229,\"text\":\"        nodes = cursor.fetchone()[0]\"},{\"line\":230,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM guests\\\")\"},{\"line\":231,\"text\":\"        guests = cursor.fetchone()[0]\"},{\"line\":232,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM services\\\")\"},{\"line\":233,\"text\":\"        services = cursor.fetchone()[0]\"},{\"line\":234,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM observations\\\")\"},{\"line\":235,\"text\":\"        observations = cursor.fetchone()[0]\"},{\"line\":236,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)\\\", (\\\"RECOVERED\\\",\\\"CLOSED\\\"))\"},{\"line\":237,\"text\":\"        incidents = cursor.fetchone()[0]\"},{\"line\":238,\"text\":\"        cursor.execute(\\\"SELECT COALESCE(EXTRACT(EPOCH FROM (now()-max(finished_at))),999999) FROM collector_runs WHERE status=%s\\\", (\\\"OK\\\",))\"},{\"line\":239,\"text\":\"        collector_age = float(cursor.fetchone()[0])\"},{\"line\":240,\"text\":\"        cursor.execute(\\\"SELECT severity,count(*) FROM incidents WHERE status NOT IN (%s,%s) GROUP BY severity ORDER BY severity\\\", (\\\"RECOVERED\\\",\\\"CLOSED\\\"))\"},{\"line\":241,\"text\":\"        severity_rows = cursor.fetchall()\"},{\"line\":242,\"text\":\"    lines = [\"},{\"line\":243,\"text\":\"        \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\"        \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\"        \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\"        \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\"        \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\"        \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\"        \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\"        \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\"        \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\"        \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\"        \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\"        \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\"        \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\"        \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\"        \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\"        \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\"        \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\"        \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\"        \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\"        \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\"        \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":264,\"text\":\"    ]\"},{\"line\":265,\"text\":\"    for severity, count in severity_rows:\"},{\"line\":266,\"text\":\"        clean = str(severity).replace(chr(34), \\\"\\\")\"},{\"line\":267,\"text\":\"        lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"},{\"line\":268,\"text\":\"    return PlainTextResponse(\\\"\\\\n\\\".join(lines) + \\\"\\\\n\\\", media_type=\\\"text/plain; version=0.0.4\\\")\"},{\"line\":269,\"text\":\"\"},{\"line\":270,\"text\":\"@contextmanager\"},{\"line\":271,\"text\":\"def api_v1_read_cursor():\"},{\"line\":272,\"text\":\"    connection = db()\"},{\"line\":273,\"text\":\"    connection.autocommit = True\"},{\"line\":274,\"text\":\"    cursor = connection.cursor(cursor_factory=RealDictCursor)\"},{\"line\":275,\"text\":\"    try:\"},{\"line\":276,\"text\":\"        cursor.execute(\\\"BEGIN READ ONLY\\\")\"},{\"line\":277,\"text\":\"        cursor.execute(\\\"SET LOCAL statement_timeout TO 5000\\\")\"},{\"line\":278,\"text\":\"        yield cursor\"},{\"line\":279,\"text\":\"    finally:\"},{\"line\":280,\"text\":\"        try:\"},{\"line\":281,\"text\":\"            cursor.execute(\\\"ROLLBACK\\\")\"},{\"line\":282,\"text\":\"        except Exception:\"},{\"line\":283,\"text\":\"            pass\"},{\"line\":284,\"text\":\"        cursor.close()\"},{\"line\":285,\"text\":\"        connection.close()\"},{\"line\":286,\"text\":\"\"},{\"line\":287,\"text\":\"\"},{\"line\":288,\"text\":\"def api_v1_require_session(request: Request):\"},{\"line\":289,\"text\":\"    data = session(request)\"},{\"line\":290,\"text\":\"    if not data:\"},{\"line\":291,\"text\":\"        raise HTTPException(401)\"},{\"line\":292,\"text\":\"    return data\"},{\"line\":293,\"text\":\"\"},{\"line\":294,\"text\":\"\"},{\"line\":295,\"text\":\"def api_v1_limit(value):\"},{\"line\":296,\"text\":\"    if value < 1 or value > 500:\"},{\"line\":297,\"text\":\"        raise HTTPException(422, detail=\\\"limit must be between 1 and 500\\\")\"},{\"line\":298,\"text\":\"    return value\"},{\"line\":299,\"text\":\"\"},{\"line\":300,\"text\":\"\"},{\"line\":301,\"text\":\"def api_v1_updated_since(value):\"},{\"line\":302,\"text\":\"    if not value:\"},{\"line\":303,\"text\":\"        return None\"},{\"line\":304,\"text\":\"    try:\"},{\"line\":305,\"text\":\"        return datetime.fromisoformat(value.replace(\\\"Z\\\", \\\"+00:00\\\"))\"},{\"line\":306,\"text\":\"    except Exception:\"},{\"line\":307,\"text\":\"        raise HTTPException(422, detail=\\\"updated_since must be ISO 8601\\\")\"},{\"line\":308,\"text\":\"\"},{\"line\":309,\"text\":\"\"},{\"line\":310,\"text\":\"def api_v1_encode_id_cursor(value):\"},{\"line\":311,\"text\":\"    return enc(str(int(value)).encode())\"},{\"line\":312,\"text\":\"\"},{\"line\":313,\"text\":\"\"},{\"line\":314,\"text\":\"def api_v1_decode_id_cursor(value):\"},{\"line\":315,\"text\":\"    if not value:\"},{\"line\":316,\"text\":\"        return None\"},{\"line\":317,\"text\":\"    try:\"},{\"line\":318,\"text\":\"        result = int(dec(value).decode())\"},{\"line\":319,\"text\":\"        if result < 0:\"},{\"line\":320,\"text\":\"            raise ValueError()\"},{\"line\":321,\"text\":\"        return result\"},{\"line\":322,\"text\":\"    except Exception:\"},{\"line\":323,\"text\":\"        raise HTTPException(422, detail=\\\"invalid cursor\\\")\"},{\"line\":324,\"text\":\"\"},{\"line\":325,\"text\":\"\"},{\"line\":326,\"text\":\"@app.get(\\\"/api/v1/entities\\\")\"},{\"line\":327,\"text\":\"def api_v1_entities(request: Request, limit: int = 100, cursor: str = \\\"\\\", entity_type: str = \\\"\\\", lifecycle_status: str = \\\"\\\", source_of_truth: str = \\\"\\\", updated_since: str = \\\"\\\", search: str = \\\"\\\"):\"},{\"line\":328,\"text\":\"    api_v1_require_session(request)\"},{\"line\":329,\"text\":\"    limit = api_v1_limit(limit)\"},{\"line\":330,\"text\":\"    clauses = []\"},{\"line\":331,\"text\":\"    params = []\"},{\"line\":332,\"text\":\"    if cursor:\"},{\"line\":333,\"text\":\"        clauses.append(\\\"e.entity_key > %s\\\")\"},{\"line\":334,\"text\":\"        params.append(cursor)\"},{\"line\":335,\"text\":\"    if entity_type:\"},{\"line\":336,\"text\":\"        clauses.append(\\\"e.entity_type = %s\\\")\"},{\"line\":337,\"text\":\"        params.append(entity_type)\"},{\"line\":338,\"text\":\"    if lifecycle_status:\"},{\"line\":339,\"text\":\"        clauses.append(\\\"e.lifecycle_status = %s\\\")\"},{\"line\":340,\"text\":\"        params.append(lifecycle_status)\"},{\"line\":341,\"text\":\"    if source_of_truth:\"},{\"line\":342,\"text\":\"        clauses.append(\\\"e.source_of_truth = %s\\\")\"},{\"line\":343,\"text\":\"        params.append(source_of_truth)\"},{\"line\":344,\"text\":\"    since = api_v1_updated_since(updated_since)\"},{\"line\":345,\"text\":\"    if since is not None:\"},{\"line\":346,\"text\":\"        clauses.append(\\\"e.updated_at >= %s\\\")\"},{\"line\":347,\"text\":\"        params.append(since)\"},{\"line\":348,\"text\":\"    if search:\"},{\"line\":349,\"text\":\"        clauses.append(\\\"(e.entity_key ILIKE %s OR e.display_name ILIKE %s)\\\")\"},{\"line\":350,\"text\":\"        params.extend([\\\"%\\\" + search + \\\"%\\\", \\\"%\\\" + search + \\\"%\\\"])\"},{\"line\":351,\"text\":\"    sql = \\\"SELECT e.entity_key,e.entity_type,e.display_name,e.lifecycle_status,e.source_of_truth,e.created_at,e.updated_at FROM entities e\\\"\"},{\"line\":352,\"text\":\"    if clauses:\"},{\"line\":353,\"text\":\"        sql += \\\" WHERE \\\" + \\\" AND \\\".join(clauses)\"},{\"line\":354,\"text\":\"    sql += \\\" ORDER BY e.entity_key LIMIT %s\\\"\"},{\"line\":355,\"text\":\"    params.append(limit + 1)\"},{\"line\":356,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":357,\"text\":\"        db_cursor.execute(sql, params)\"},{\"line\":358,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":359,\"text\":\"    page = rows[:limit]\"},{\"line\":360,\"text\":\"    next_cursor = page[-1][\\\"entity_key\\\"] if len(rows) > limit else None\"},{\"line\":361,\"text\":\"    return {\\\"items\\\": page, \\\"next_cursor\\\": next_cursor, \\\"limit\\\": limit}\"},{\"line\":362,\"text\":\"\"},{\"line\":363,\"text\":\"\"},{\"line\":364,\"text\":\"@app.get(\\\"/api/v1/entities/{entity_key:path}/dependencies\\\")\"},{\"line\":365,\"text\":\"def api_v1_entity_dependencies(request: Request, entity_key: str, limit: int = 100, cursor: str = \\\"\\\"):\"},{\"line\":366,\"text\":\"    api_v1_require_session(request)\"},{\"line\":367,\"text\":\"    limit = api_v1_limit(limit)\"},{\"line\":368,\"text\":\"    cursor_id = api_v1_decode_id_cursor(cursor)\"},{\"line\":369,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":370,\"text\":\"        db_cursor.execute(\\\"SELECT id FROM entities WHERE entity_key=%s\\\", (entity_key,))\"},{\"line\":371,\"text\":\"        entity = db_cursor.fetchone()\"},{\"line\":372,\"text\":\"        if not entity:\"},{\"line\":373,\"text\":\"            raise HTTPException(404, detail=\\\"entity not found\\\")\"},{\"line\":374,\"text\":\"        entity_id = entity[\\\"id\\\"]\"},{\"line\":375,\"text\":\"        clauses = [\\\"(d.parent_entity_id=%s OR d.child_entity_id=%s)\\\"]\"},{\"line\":376,\"text\":\"        params = [entity_id, entity_id, entity_id]\"},{\"line\":377,\"text\":\"        if cursor_id is not None:\"},{\"line\":378,\"text\":\"            clauses.append(\\\"d.id > %s\\\")\"},{\"line\":379,\"text\":\"            params.append(cursor_id)\"},{\"line\":380,\"text\":\"        sql = \\\"SELECT d.id AS _cursor_id,CASE WHEN d.parent_entity_id=%s THEN 'outgoing' ELSE 'incoming' END AS direction,p.entity_key AS parent_entity_key,c.entity_key AS child_entity_key,d.relation_type,d.source_relation,d.criticality,d.edge_state,d.created_at,d.updated_at FROM entity_dependencies d JOIN entities p ON p.id=d.parent_entity_id JOIN entities c ON c.id=d.child_entity_id WHERE \\\" + \\\" AND \\\".join(clauses) + \\\" ORDER BY d.id LIMIT %s\\\"\"},{\"line\":381,\"text\":\"        params.append(limit + 1)\"},{\"line\":382,\"text\":\"        db_cursor.execute(sql, params)\"},{\"line\":383,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":384,\"text\":\"    page = rows[:limit]\"},{\"line\":385,\"text\":\"    next_cursor = api_v1_encode_id_cursor(page[-1][\\\"_cursor_id\\\"]) if len(rows) > limit else None\"},{\"line\":386,\"text\":\"    for row in page:\"},{\"line\":387,\"text\":\"        row.pop(\\\"_cursor_id\\\", None)\"},{\"line\":388,\"text\":\"    return {\\\"entity_key\\\": entity_key, \\\"items\\\": page, \\\"next_cursor\\\": next_cursor, \\\"limit\\\": limit}\"},{\"line\":389,\"text\":\"\"},{\"line\":390,\"text\":\"\"},{\"line\":391,\"text\":\"@app.get(\\\"/api/v1/entities/{entity_key:path}\\\")\"},{\"line\":392,\"text\":\"def api_v1_entity_detail(request: Request, entity_key: str):\"},{\"line\":393,\"text\":\"    api_v1_require_session(request)\"},{\"line\":394,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":395,\"text\":\"        db_cursor.execute(\\\"SELECT id AS _entity_id,entity_key,entity_type,display_name,lifecycle_status,source_of_truth,legacy_table,created_at,updated_at FROM entities WHERE entity_key=%s\\\", (entity_key,))\"},{\"line\":396,\"text\":\"        row = db_cursor.fetchone()\"},{\"line\":397,\"text\":\"        if not row:\"},{\"line\":398,\"text\":\"            raise HTTPException(404, detail=\\\"entity not found\\\")\"},{\"line\":399,\"text\":\"        result = dict(row)\"},{\"line\":400,\"text\":\"        entity_id = result.pop(\\\"_entity_id\\\")\"},{\"line\":401,\"text\":\"        db_cursor.execute(\\\"SELECT alias,alias_type,priority,is_canonical,created_at FROM entity_aliases WHERE entity_id=%s ORDER BY is_canonical DESC,priority,alias,alias_type\\\", (entity_id,))\"},{\"line\":402,\"text\":\"        result[\\\"aliases\\\"] = [dict(alias) for alias in db_cursor.fetchall()]\"},{\"line\":403,\"text\":\"    return result\"},{\"line\":404,\"text\":\"\"},{\"line\":405,\"text\":\"\"},{\"line\":406,\"text\":\"@app.get(\\\"/api/v1/dependencies\\\")\"},{\"line\":407,\"text\":\"def api_v1_dependencies(request: Request, limit: int = 100, cursor: str = \\\"\\\", parent_entity_key: str = \\\"\\\", child_entity_key: str = \\\"\\\", relation_type: str = \\\"\\\", source_relation: str = \\\"\\\", criticality: str = \\\"\\\", edge_state: str = \\\"\\\"):\"},{\"line\":408,\"text\":\"    api_v1_require_session(request)\"},{\"line\":409,\"text\":\"    limit = api_v1_limit(limit)\"},{\"line\":410,\"text\":\"    cursor_id = api_v1_decode_id_cursor(cursor)\"},{\"line\":411,\"text\":\"    clauses = []\"},{\"line\":412,\"text\":\"    params = []\"},{\"line\":413,\"text\":\"    if cursor_id is not None:\"},{\"line\":414,\"text\":\"        clauses.append(\\\"d.id > %s\\\")\"},{\"line\":415,\"text\":\"        params.append(cursor_id)\"},{\"line\":416,\"text\":\"    filters = [(\\\"p.entity_key\\\", parent_entity_key), (\\\"c.entity_key\\\", child_entity_key), (\\\"d.relation_type\\\", relation_type), (\\\"d.source_relation\\\", source_relation), (\\\"d.criticality\\\", criticality), (\\\"d.edge_state\\\", edge_state)]\"},{\"line\":417,\"text\":\"    for column, value in filters:\"},{\"line\":418,\"text\":\"        if value:\"},{\"line\":419,\"text\":\"            clauses.append(column + \\\" = %s\\\")\"},{\"line\":420,\"text\":\"            params.append(value)\"},{\"line\":421,\"text\":\"    sql = \\\"SELECT d.id AS _cursor_id,p.entity_key AS parent_entity_key,c.entity_key AS child_entity_key,p.entity_type AS parent_entity_type,c.entity_type AS child_entity_type,d.relation_type,d.source_relation,d.criticality,d.edge_state,d.created_at,d.updated_at FROM entity_dependencies d JOIN entities p ON p.id=d.parent_entity_id JOIN entities c ON c.id=d.child_entity_id\\\"\"},{\"line\":422,\"text\":\"    if clauses:\"},{\"line\":423,\"text\":\"        sql += \\\" WHERE \\\" + \\\" AND \\\".join(clauses)\"},{\"line\":424,\"text\":\"    sql += \\\" ORDER BY d.id LIMIT %s\\\"\"},{\"line\":425,\"text\":\"    params.append(limit + 1)\"},{\"line\":426,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":427,\"text\":\"        db_cursor.execute(sql, params)\"},{\"line\":428,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":429,\"text\":\"    page = rows[:limit]\"},{\"line\":430,\"text\":\"    next_cursor = api_v1_encode_id_cursor(page[-1][\\\"_cursor_id\\\"]) if len(rows) > limit else None\"},{\"line\":431,\"text\":\"    for row in page:\"},{\"line\":432,\"text\":\"        row.pop(\\\"_cursor_id\\\", None)\"},{\"line\":433,\"text\":\"    return {\\\"items\\\": page, \\\"next_cursor\\\": next_cursor, \\\"limit\\\": limit}\"},{\"line\":434,\"text\":\"\"},{\"line\":435,\"text\":\"\"},{\"line\":436,\"text\":\"@app.get(\\\"/api/v1/sources\\\")\"},{\"line\":437,\"text\":\"def api_v1_sources(request: Request):\"},{\"line\":438,\"text\":\"    api_v1_require_session(request)\"},{\"line\":439,\"text\":\"    sql = \\\"SELECT s.source_key,s.source_type,s.enabled,si.instance_key,si.status AS instance_status,si.last_seen,se.expectation_type,se.interval_seconds,se.grace_seconds,se.severity,s.created_at,s.updated_at FROM sources s LEFT JOIN source_instances si ON si.source_id=s.id LEFT JOIN source_expectations se ON se.source_instance_id=si.id ORDER BY s.source_key,si.instance_key,se.expectation_type\\\"\"},{\"line\":440,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":441,\"text\":\"        db_cursor.execute(sql)\"},{\"line\":442,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":443,\"text\":\"    return {\\\"items\\\": rows}\"},{\"line\":444,\"text\":\"\"},{\"line\":445,\"text\":\"\"},{\"line\":446,\"text\":\"@app.get(\\\"/api/v1/policies\\\")\"},{\"line\":447,\"text\":\"def api_v1_policies(request: Request):\"},{\"line\":448,\"text\":\"    api_v1_require_session(request)\"},{\"line\":449,\"text\":\"    sql = \\\"SELECT pr.policy_key,pr.scope,pr.enabled,pr.active_version,pv.document->>'mode' AS mode,pv.document->>'automatic_actions' AS automatic_actions,pv.document->>'human_approval_required' AS human_approval_required,pr.created_at,pr.updated_at FROM policy_registry pr LEFT JOIN policy_versions pv ON pv.policy_id=pr.id AND pv.version=pr.active_version ORDER BY pr.policy_key,pr.scope\\\"\"},{\"line\":450,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":451,\"text\":\"        db_cursor.execute(sql)\"},{\"line\":452,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":453,\"text\":\"    return {\\\"items\\\": rows}\"},{\"line\":454,\"text\":\"\"},{\"line\":455,\"text\":\"\"},{\"line\":456,\"text\":\"@app.get(\\\"/api/v1/collector-runs\\\")\"},{\"line\":457,\"text\":\"def api_v1_collector_runs(request: Request, limit: int = 100, cursor: str = \\\"\\\", collector: str = \\\"\\\", status: str = \\\"\\\"):\"},{\"line\":458,\"text\":\"    api_v1_require_session(request)\"},{\"line\":459,\"text\":\"    limit = api_v1_limit(limit)\"},{\"line\":460,\"text\":\"    cursor_id = api_v1_decode_id_cursor(cursor)\"},{\"line\":461,\"text\":\"    clauses = []\"},{\"line\":462,\"text\":\"    params = []\"},{\"line\":463,\"text\":\"    if cursor_id is not None:\"},{\"line\":464,\"text\":\"        clauses.append(\\\"cr.id < %s\\\")\"},{\"line\":465,\"text\":\"        params.append(cursor_id)\"},{\"line\":466,\"text\":\"    if collector:\"},{\"line\":467,\"text\":\"        clauses.append(\\\"cr.collector = %s\\\")\"},{\"line\":468,\"text\":\"        params.append(collector)\"},{\"line\":469,\"text\":\"    if status:\"},{\"line\":470,\"text\":\"        clauses.append(\\\"cr.status = %s\\\")\"},{\"line\":471,\"text\":\"        params.append(status)\"},{\"line\":472,\"text\":\"    sql = \\\"SELECT cr.id AS _cursor_id,cr.collector,cr.status,cr.trigger_kind,cr.schema_version,cr.records_seen,cr.warning_count,cr.run_key,si.instance_key AS source_instance_key,cr.started_at,cr.finished_at FROM collector_runs cr LEFT JOIN source_instances si ON si.id=cr.source_instance_id\\\"\"},{\"line\":473,\"text\":\"    if clauses:\"},{\"line\":474,\"text\":\"        sql += \\\" WHERE \\\" + \\\" AND \\\".join(clauses)\"},{\"line\":475,\"text\":\"    sql += \\\" ORDER BY cr.id DESC LIMIT %s\\\"\"},{\"line\":476,\"text\":\"    params.append(limit + 1)\"},{\"line\":477,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":478,\"text\":\"        db_cursor.execute(sql, params)\"},{\"line\":479,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":480,\"text\":\"    page = rows[:limit]\"},{\"line\":481,\"text\":\"    next_cursor = api_v1_encode_id_cursor(page[-1][\\\"_cursor_id\\\"]) if len(rows) > limit else None\"},{\"line\":482,\"text\":\"    for row in page:\"},{\"line\":483,\"text\":\"        row.pop(\\\"_cursor_id\\\", None)\"},{\"line\":484,\"text\":\"    return {\\\"items\\\": page, \\\"next_cursor\\\": next_cursor, \\\"limit\\\": limit}\"}]},{\"bytes\":10640,\"line_count\":113,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"priority\":50,\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env python3\"},{\"line\":2,\"text\":\"import datetime\"},{\"line\":3,\"text\":\"import json\"},{\"line\":4,\"text\":\"import re\"},{\"line\":5,\"text\":\"import urllib.parse\"},{\"line\":6,\"text\":\"import urllib.request\"},{\"line\":7,\"text\":\"import psycopg2\"},{\"line\":8,\"text\":\"from psycopg2.extras import Json\"},{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":11,\"text\":\"PROMETHEUS = \\\"http://[PRIVATE_IP]:9090\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":13,\"text\":\"def prometheus(expression):\"},{\"line\":14,\"text\":\"    try:\"},{\"line\":15,\"text\":\"        url = PROMETHEUS + \\\"/api/v1/query?\\\" + urllib.parse.urlencode({\\\"query\\\":expression})\"},{\"line\":16,\"text\":\"        with urllib.request.urlopen(url, timeout=12) as response:\"},{\"line\":17,\"text\":\"            body = json.loads(response.read().decode(\\\"utf-8\\\"))\"},{\"line\":18,\"text\":\"        return body.get(\\\"data\\\", {}).get(\\\"result\\\", []) if body.get(\\\"status\\\") == \\\"success\\\" else []\"},{\"line\":19,\"text\":\"    except Exception:\"},{\"line\":20,\"text\":\"        return []\"},{\"line\":21,\"text\":\"def numeric(value):\"},{\"line\":22,\"text\":\"    try:\"},{\"line\":23,\"text\":\"        return float(value)\"},{\"line\":24,\"text\":\"    except Exception:\"},{\"line\":25,\"text\":\"        return None\"},{\"line\":26,\"text\":\"def slug(value):\"},{\"line\":27,\"text\":\"    return re.sub(r\\\"[^a-z0-9]+\\\", \\\"-\\\", str(value).lower()).strip(\\\"-\\\")\"},{\"line\":28,\"text\":\"def entity(instance):\"},{\"line\":29,\"text\":\"    address = instance.rsplit(\\\":\\\", 1)[0]\"},{\"line\":30,\"text\":\"    return HOSTS.get(address, address)\"},{\"line\":31,\"text\":\"def open_incident(cursor, fingerprint, severity, title, summary, cause, metadata):\"},{\"line\":32,\"text\":\"    cursor.execute(\\\"INSERT INTO incidents(fingerprint,status,severity,title,summary,root_cause,first_seen,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,now(),now(),%s) ON CONFLICT(fingerprint) DO UPDATE SET status=EXCLUDED.status,severity=EXCLUDED.severity,title=EXCLUDED.title,summary=EXCLUDED.summary,root_cause=EXCLUDED.root_cause,last_seen=now(),recovered_at=NULL,metadata=EXCLUDED.metadata RETURNING id\\\",(fingerprint,\\\"OPEN\\\",severity,title,summary,cause,Json(metadata)))\"},{\"line\":33,\"text\":\"    incident_id = cursor.fetchone()[0]\"},{\"line\":34,\"text\":\"    cursor.execute(\\\"INSERT INTO incident_events(incident_id,event_type,message,metadata) SELECT %s,%s,%s,%s WHERE NOT EXISTS(SELECT 1 FROM incident_events WHERE incident_id=%s AND event_type=%s AND created_at>now()-make_interval(mins=>10))\\\",(incident_id,\\\"OBSERVED\\\",summary,Json(metadata),incident_id,\\\"OBSERVED\\\"))\"},{\"line\":35,\"text\":\"def main():\"},{\"line\":36,\"text\":\"    data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\"    connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":38,\"text\":\"    active = set()\"},{\"line\":39,\"text\":\"    metric_count = 0\"},{\"line\":40,\"text\":\"    with connection.cursor() as cursor:\"},{\"line\":41,\"text\":\"        cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":42,\"text\":\"        run_id = cursor.fetchone()[0]\"},{\"line\":43,\"text\":\"        for item in data.get(\\\"nodes\\\", []):\"},{\"line\":44,\"text\":\"            cursor.execute(\\\"INSERT INTO nodes(name,address,role,status,last_seen,metadata) VALUES(%s,%s,%s,%s,now(),%s) ON CONFLICT(name) DO UPDATE SET address=EXCLUDED.address,role=EXCLUDED.role,status=EXCLUDED.status,last_seen=now(),metadata=EXCLUDED.metadata\\\",(item.get(\\\"name\\\"),item.get(\\\"address\\\"),item.get(\\\"role\\\"),item.get(\\\"status\\\"),Json(item)))\"},{\"line\":45,\"text\":\"        for item in data.get(\\\"guests\\\", []):\"},{\"line\":46,\"text\":\"            cursor.execute(\\\"INSERT INTO guests(guest_key,node_name,guest_type,name,status,cpu_usage,memory_used_bytes,memory_total_bytes,disk_used_bytes,disk_total_bytes,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,now(),%s) ON CONFLICT(guest_key) DO UPDATE SET node_name=EXCLUDED.node_name,guest_type=EXCLUDED.guest_type,name=EXCLUDED.name,status=EXCLUDED.status,cpu_usage=EXCLUDED.cpu_usage,memory_used_bytes=EXCLUDED.memory_used_bytes,memory_total_bytes=EXCLUDED.memory_total_bytes,disk_used_bytes=EXCLUDED.disk_used_bytes,disk_total_bytes=EXCLUDED.disk_total_bytes,last_seen=now(),metadata=EXCLUDED.metadata\\\",(item.get(\\\"guest_key\\\"),item.get(\\\"node_name\\\"),item.get(\\\"guest_type\\\"),item.get(\\\"name\\\"),item.get(\\\"status\\\"),item.get(\\\"cpu_usage\\\"),item.get(\\\"memory_used_bytes\\\"),item.get(\\\"memory_total_bytes\\\"),item.get(\\\"disk_used_bytes\\\"),item.get(\\\"disk_total_bytes\\\"),Json(item)))\"},{\"line\":47,\"text\":\"        for item in data.get(\\\"services\\\", []):\"},{\"line\":48,\"text\":\"            service_key = item.get(\\\"service_key\\\") or slug(item.get(\\\"name\\\"))\"},{\"line\":49,\"text\":\"            cursor.execute(\\\"INSERT INTO services(service_key,name,scope,host_name,container_name,public_url,internal_url,status,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,now(),%s) ON CONFLICT(service_key) DO UPDATE SET name=EXCLUDED.name,scope=EXCLUDED.scope,host_name=EXCLUDED.host_name,container_name=EXCLUDED.container_name,public_url=EXCLUDED.public_url,internal_url=EXCLUDED.internal_url,status=EXCLUDED.status,last_seen=now(),metadata=EXCLUDED.metadata\\\",(service_key,item.get(\\\"name\\\"),item.get(\\\"scope\\\"),item.get(\\\"host_name\\\"),item.get(\\\"container_name\\\"),item.get(\\\"public_url\\\"),item.get(\\\"internal_url\\\"),item.get(\\\"status\\\"),Json(item)))\"},{\"line\":50,\"text\":\"            if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":51,\"text\":\"                fingerprint = \\\"service:\\\" + service_key\"},{\"line\":52,\"text\":\"                active.add(fingerprint)\"},{\"line\":53,\"text\":\"                open_incident(cursor,fingerprint,\\\"P1\\\",\\\"Service requires attention: \\\" + str(item.get(\\\"name\\\")),\\\"Readiness status is \\\" + str(item.get(\\\"status\\\")),\\\"service-readiness\\\",item)\"},{\"line\":54,\"text\":\"        cursor.execute(\\\"DELETE FROM dependencies\\\")\"},{\"line\":55,\"text\":\"        for item in data.get(\\\"dependencies\\\", []):\"},{\"line\":56,\"text\":\"            cursor.execute(\\\"INSERT INTO dependencies(parent_key,child_key,relation,metadata) VALUES(%s,%s,%s,%s) ON CONFLICT(parent_key,child_key,relation) DO UPDATE SET metadata=EXCLUDED.metadata\\\",(item.get(\\\"parent_key\\\"),item.get(\\\"child_key\\\"),item.get(\\\"relation\\\"),Json(item)))\"},{\"line\":57,\"text\":\"        cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":58,\"text\":\"        expressions = {\\\"up\\\":\\\"up\\\",\\\"cpu_used_pct\\\":\\\"100-(avg by(instance)(rate(node_cpu_seconds_total{mode=\\\\\\\"idle\\\\\\\"}[5m]))*100)\\\",\\\"memory_used_pct\\\":\\\"100*(1-(node_memory_MemAvailable_bytes/node_memory_MemTotal_bytes))\\\",\\\"filesystem_used_pct\\\":\\\"100*(1-(node_filesystem_avail_bytes{fstype!~\\\\\\\"tmpfs|overlay|squashfs|nsfs\\\\\\\"}/node_filesystem_size_bytes{fstype!~\\\\\\\"tmpfs|overlay|squashfs|nsfs\\\\\\\"}))\\\",\\\"load1\\\":\\\"node_load1\\\"}\"},{\"line\":59,\"text\":\"        for metric, expression in expressions.items():\"},{\"line\":60,\"text\":\"            for result in prometheus(expression):\"},{\"line\":61,\"text\":\"                labels = result.get(\\\"metric\\\", {})\"},{\"line\":62,\"text\":\"                instance = labels.get(\\\"instance\\\", \\\"unknown\\\")\"},{\"line\":63,\"text\":\"                entity_key = entity(instance)\"},{\"line\":64,\"text\":\"                value = numeric(result.get(\\\"value\\\", [None, None])[1])\"},{\"line\":65,\"text\":\"                if value is None:\"},{\"line\":66,\"text\":\"                    continue\"},{\"line\":67,\"text\":\"                status = \\\"OK\\\"\"},{\"line\":68,\"text\":\"                severity = \\\"P2\\\"\"},{\"line\":69,\"text\":\"                if metric == \\\"up\\\" and value < 1:\"},{\"line\":70,\"text\":\"                    status, severity = \\\"BAD\\\", \\\"P1\\\"\"},{\"line\":71,\"text\":\"                elif metric == \\\"cpu_used_pct\\\" and value >= 97:\"},{\"line\":72,\"text\":\"                    status, severity = \\\"BAD\\\", \\\"P1\\\"\"},{\"line\":73,\"text\":\"                elif metric == \\\"cpu_used_pct\\\" and value >= 90:\"},{\"line\":74,\"text\":\"                    status = \\\"WARN\\\"\"},{\"line\":75,\"text\":\"                elif metric == \\\"memory_used_pct\\\" and value >= 95:\"},{\"line\":76,\"text\":\"                    status, severity = \\\"BAD\\\", \\\"P1\\\"\"},{\"line\":77,\"text\":\"                elif metric == \\\"memory_used_pct\\\" and value >= 88:\"},{\"line\":78,\"text\":\"                    status = \\\"WARN\\\"\"},{\"line\":79,\"text\":\"                elif metric == \\\"filesystem_used_pct\\\" and value >= 95:\"},{\"line\":80,\"text\":\"                    status, severity = \\\"BAD\\\", \\\"P1\\\"\"},{\"line\":81,\"text\":\"                elif metric == \\\"filesystem_used_pct\\\" and value >= 85:\"},{\"line\":82,\"text\":\"                    status = \\\"WARN\\\"\"},{\"line\":83,\"text\":\"                unit = \\\"percent\\\" if metric.endswith(\\\"pct\\\") else \\\"state\\\" if metric == \\\"up\\\" else \\\"load\\\"\"},{\"line\":84,\"text\":\"                metadata = {\\\"instance\\\":instance,\\\"job\\\":labels.get(\\\"job\\\"),\\\"mountpoint\\\":labels.get(\\\"mountpoint\\\"),\\\"device\\\":labels.get(\\\"device\\\")}\"},{\"line\":85,\"text\":\"                cursor.execute(\\\"INSERT INTO observations(entity_type,entity_key,metric,value,unit,status,observed_at,metadata) VALUES(%s,%s,%s,%s,%s,%s,now(),%s)\\\",(\\\"target\\\",entity_key,metric,value,unit,status,Json(metadata)))\"},{\"line\":86,\"text\":\"                metric_count += 1\"},{\"line\":87,\"text\":\"                if status != \\\"OK\\\":\"},{\"line\":88,\"text\":\"                    cursor.execute(\\\"SELECT count(*) FROM observations WHERE entity_type=%s AND entity_key=%s AND metric=%s AND status=ANY(%s) AND observed_at>now()-make_interval(mins=>10)\\\",(\\\"target\\\",entity_key,metric,[\\\"WARN\\\",\\\"BAD\\\"]))\"},{\"line\":89,\"text\":\"                    if cursor.fetchone()[0] >= 2:\"},{\"line\":90,\"text\":\"                        fingerprint = \\\"metric:\\\" + slug(entity_key) + \\\":\\\" + metric + \\\":\\\" + slug(labels.get(\\\"mountpoint\\\", \\\"root\\\"))\"},{\"line\":91,\"text\":\"                        active.add(fingerprint)\"},{\"line\":92,\"text\":\"                        open_incident(cursor,fingerprint,severity,entity_key + \\\" - \\\" + metric,metric + \\\"=\\\" + format(value, \\\".2f\\\") + \\\" \\\" + unit,\\\"target-unavailable\\\" if metric == \\\"up\\\" else \\\"resource-pressure\\\",metadata)\"},{\"line\":93,\"text\":\"        for name, state in data.get(\\\"backups\\\", {}).items():\"},{\"line\":94,\"text\":\"            if state.get(\\\"status\\\") != \\\"success\\\" or not state.get(\\\"local_backup_deleted\\\", False) or not state.get(\\\"remote_a_job_path\\\") or not state.get(\\\"remote_b_job_path\\\"):\"},{\"line\":95,\"text\":\"                fingerprint = \\\"backup:\\\" + slug(name)\"},{\"line\":96,\"text\":\"                active.add(fingerprint)\"},{\"line\":97,\"text\":\"                open_incident(cursor,fingerprint,\\\"P1\\\",\\\"Backup requires attention: \\\" + name,\\\"Backup is not fully verified\\\",\\\"backup-verification\\\",state)\"},{\"line\":98,\"text\":\"        cursor.execute(\\\"SELECT id,fingerprint FROM incidents WHERE status=ANY(%s)\\\",([\\\"OPEN\\\",\\\"DETECTED\\\",\\\"ACKNOWLEDGED\\\",\\\"MONITORING\\\"],))\"},{\"line\":99,\"text\":\"        for incident_id, fingerprint in cursor.fetchall():\"},{\"line\":100,\"text\":\"            if fingerprint not in active:\"},{\"line\":101,\"text\":\"                cursor.execute(\\\"UPDATE incidents SET status=%s,recovered_at=now(),last_seen=now() WHERE id=%s\\\",(\\\"RECOVERED\\\",incident_id))\"},{\"line\":102,\"text\":\"                cursor.execute(\\\"INSERT INTO incident_events(incident_id,event_type,message,metadata) VALUES(%s,%s,%s,%s)\\\",(incident_id,\\\"RECOVERED\\\",\\\"Condition returned to normal\\\",Json({})))\"},{\"line\":103,\"text\":\"        cursor.execute(\\\"DELETE FROM observations WHERE observed_at<now()-make_interval(days=>30)\\\")\"},{\"line\":104,\"text\":\"        cursor.execute(\\\"UPDATE collector_runs SET status=%s,finished_at=now(),metadata=%s WHERE id=%s\\\",(\\\"OK\\\",Json({\\\"nodes\\\":len(data.get(\\\"nodes\\\",[])),\\\"guests\\\":len(data.get(\\\"guests\\\",[])),\\\"services\\\":len(data.get(\\\"services\\\",[])),\\\"metrics\\\":metric_count}),run_id))\"},{\"line\":105,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM incidents WHERE status=ANY(%s)\\\",([\\\"OPEN\\\",\\\"DETECTED\\\",\\\"ACKNOWLEDGED\\\",\\\"MONITORING\\\"],))\"},{\"line\":106,\"text\":\"        open_count = cursor.fetchone()[0]\"},{\"line\":107,\"text\":\"    connection.commit()\"},{\"line\":108,\"text\":\"    line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"},{\"line\":109,\"text\":\"    with open(HEALTH, \\\"w\\\", encoding=\\\"utf-8\\\") as stream:\"},{\"line\":110,\"text\":\"        stream.write(line)\"},{\"line\":111,\"text\":\"    print(line.strip())\"},{\"line\":112,\"text\":\"if __name__ == \\\"__main__\\\":\"},{\"line\":113,\"text\":\"    main()\"}]}],\"gate\":{\"exact_source_count\":30,\"index_source_count\":1,\"ready_for_atomic_design\":true,\"reference_source_count\":1,\"unit_record_count\":16},\"hostname\":\"cluster-admin\",\"index_paths\":[\"/var/www/homelab-cluster-admin/index.html\"],\"input_counts\":{\"application_candidates\":40,\"health_input_files\":4,\"indexes\":1,\"reference_files\":1,\"units\":16},\"reference_paths\":[\"/opt/homelab-cluster-admin/render-webpanel.sh\"],\"unit_records\":[{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006178 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:27 UTC] ; stop_time=[Wed 2026-08-05 13:20:28 UTC] ; pid=2006164 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:15 UTC] ; pid=2005776 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005777 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005778 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006179 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:11:27 UTC] ; stop_time=[Wed 2026-08-05 13:11:27 UTC] ; pid=2004608 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"Group\":\"root\",\"LoadState\":\"loaded\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"root\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}]},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":18,\"status\":\"OK\",\"transaction_gate\":{\"index_source_found\":true,\"next_required_action\":\"build corrected atomic local+remote transaction\",\"ready_for_atomic_transaction\":true,\"reference_source_found\":true,\"remote_probe_root\":true,\"unit_records_found\":true},\"transport\":{\"attempts\":[{\"privilege\":\"sudo-root\",\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":139487,\"stdout_sha256\":\"a710627984d38927f5e0fdde30c8beab966ad52581868b9928ea458dc6902bf3\"}],\"privilege\":\"sudo-root\",\"success\":true},\"v15_candidate_manifest\":{\"application_candidates\":[{\"bytes\":133,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},{\"bytes\":149,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},{\"bytes\":151,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},{\"bytes\":231,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},{\"bytes\":138,\"matches\":[{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},{\"bytes\":127,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},{\"bytes\":616,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},{\"bytes\":1190,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},{\"bytes\":223,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},{\"bytes\":803,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},{\"bytes\":131,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},{\"bytes\":301,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},{\"bytes\":89,\"matches\":[{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\"},{\"bytes\":8444,\"matches\":[{\"line\":4,\"text\":\"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')\"},{\"line\":64,\"text\":\" return f\\\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\\\n\\\"\"},{\"line\":82,\"text\":\" old=<REDACTED>\"},{\"line\":97,\"text\":\"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)\"},{\"line\":99,\"text\":\"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\"},{\"bytes\":28599,\"matches\":[{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =<REDACTED>\"},{\"line\":14,\"text\":\"COOKIE =<REDACTED>\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":20,\"text\":\"    return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":73,\"text\":\"    return f\\\"\\\"\\\"<!doctype html><html lang=<REDACTED>\"},{\"line\":152,\"text\":\"    return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":220,\"text\":\"    page = f\\\"\\\"\\\"<!doctype html><html lang=\\\"ru\\\"><head><meta charset=\\\"utf-8\\\"><meta name=\\\"viewport\\\" content=\\\"width=device-width,initial-scale=1\\\"><meta http-equiv=\\\"refresh\\\" content=\\\"30\\\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\\\"post\\\" action=\\\"/logout\\\"><input type=\\\"hidden\\\" name=\\\"csrf\\\" value=\\\"{csrf}\\\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\\\"\\\"\\\"\"},{\"line\":243,\"text\":\"        \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\"        \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\"        \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\"        \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\"        \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\"        \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\"        \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\"        \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\"        \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\"        \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\"        \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\"        \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\"        \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\"        \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\"        \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\"        \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\"        \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\"        \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\"        \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\"        \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\"        \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":267,\"text\":\"        lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\"},{\"bytes\":922,\"matches\":[{\"line\":2,\"text\":\"first = getpass.getpass(\\\"Новый пароль Cluster Admin: \\\")\"},{\"line\":12,\"text\":\"directory = \\\"/etc/cluster-admin-incident-engine\\\"\"},{\"line\":16,\"text\":\"account = pwd.getpwnam(\\\"clusteradmin\\\")\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\"},{\"bytes\":10640,\"matches\":[{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":36,\"text\":\"    data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\"    connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":41,\"text\":\"        cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":50,\"text\":\"            if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":57,\"text\":\"        cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":108,\"text\":\"    line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\"},{\"bytes\":1857,\"matches\":[{\"line\":3,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage1-foundation-migration-plan\\\",\"},{\"line\":43,\"text\":\"    \\\"function\\\": \\\"cluster_admin_enrich_collector_run\\\",\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\"},{\"bytes\":1342,\"matches\":[{\"line\":3,\"text\":\"  \\\"collector_call\\\": \\\"SELECT cluster_admin_sync_foundation(%s)\\\",\"},{\"line\":15,\"text\":\"  \\\"function\\\": \\\"cluster_admin_sync_foundation\\\",\"},{\"line\":16,\"text\":\"  \\\"function_signature\\\": \\\"cluster_admin_sync_foundation(bigint)\\\",\"},{\"line\":26,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-migration-plan\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\"},{\"bytes\":2355,\"matches\":[{\"line\":51,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-dynamic-acceptance-contract\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\"},{\"bytes\":2832,\"matches\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\"  line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\"  echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},{\"bytes\":3471,\"matches\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\"  echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\"  echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},{\"bytes\":1433,\"matches\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},{\"bytes\":848,\"matches\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":10353,\"matches\":[{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\"},{\"bytes\":168,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729\"},{\"bytes\":259,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741\"},{\"bytes\":134,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517\"},{\"bytes\":141,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\"},{\"bytes\":189,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\"},{\"bytes\":131,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":199,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\"},{\"bytes\":272,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec\"},{\"bytes\":380,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"}],\"gate\":{\"application_candidate_count\":40,\"ready_for_remote_transaction_design\":true,\"reference_file_count\":1,\"unit_count\":16},\"health_input_files\":[{\"bytes\":380,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"},{\"bytes\":128,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\"},{\"bytes\":1535,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\"}],\"indexes\":[{\"bytes\":10353,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"root\":\"/var/www/homelab-cluster-admin\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\",\"title_lines\":[\"<title>Homelab Cluster Admin</title>\",\"<h1>Homelab Cluster Admin</h1>\",\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\",\"<div class=<REDACTED>\",\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\",\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"]}],\"listeners\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"State  Recv-Q Send-Q Local Address:Port Peer Address:PortProcess                                   \\nLISTEN 0      5            0.0.0.0:8080      0.0.0.0:*    users:((\\\"python3\\\",pid=439,fd=3))         \\nLISTEN 0      2048         0.0.0.0:8081      0.0.0.0:*    users:((\\\"python3\\\",pid=487,fd=6))         \\nLISTEN 0      244        127.0.0.1:5432      0.0.0.0:*    users:((\\\"postgres\\\",pid=469,fd=6))        \\nLISTEN 0      4096   127.0.0.53%lo:53        0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=18))\\nLISTEN 0      4096         0.0.0.0:5355      0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=12))\\nLISTEN 0      4096      127.0.0.54:53        0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=20))\\nLISTEN 0      128          0.0.0.0:22        0.0.0.0:*    users:((\\\"sshd\\\",pid=457,fd=3))            \\nLISTEN 0      244            [::1]:5432         [::]:*    users:((\\\"postgres\\\",pid=469,fd=5))        \\nLISTEN 0      4096            [::]:5355         [::]:*    users:((\\\"systemd-resolve\\\",pid=380,fd=14))\\nLISTEN 0      128             [::]:22           [::]:*    users:((\\\"sshd\\\",pid=457,fd=4))            \"},\"process_lines\":[\"    439       1 root     root     python3         /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\",\"    487       1 cluster+ cluster+ python3         /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\"],\"reference_files\":[{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"}],\"units\":[{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"Group\":\"root\",\"LoadState\":\"loaded\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"root\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}]},\"v15_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"declared_report_bytes\":85056,\"declared_report_sha256\":\"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf\",\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":85940,\"output_sha256\":\"c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8\",\"output_source\":\"runner_json\",\"rollback_restored\":null,\"rollback_started\":false,\"runner_document_rc\":0,\"runner_json_sha256\":\"6769d194cb102deb6fc37293d887af601bd012448bac4de49dd4bca3301f34a7\",\"runner_rc\":0,\"runner_status\":\"OK\",\"runner_text_sha256\":\"c68484da49e0164adaccfe9ecf92120c53d9324b2c3dcb96b4088376d3d177d2\",\"runner_wrapper_matches_document\":true,\"sanitized_report_bytes\":85006,\"sanitized_report_sha256\":\"f05595551c37c00d473df29113cf9d597b2d3467ad5a2b50e8803e8579accc82\",\"sanitizer_byte_delta\":50,\"sanitizer_changed_report\":true,\"state_document_sha256\":\"f438ecd342026e3db3f910ff87aba2e3c746147cf3fa64217fc4317774055874\",\"state_path_sha256\":\"993095398c2c97878a69cd2e0aaa11e2ae1f587df8c8331835b5fd7ab7aa9de7\",\"state_status\":\"OK\"}}\n"
}