CHAT_OUTPUT_BEGIN
COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z
STATUS=OK
RC=0
HOST=pve01
MODE=verify
COMPONENT=homelab-cluster-admin-update-visibility
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
COMMAND_SHA256=8e9116b7bd5174aea8762e6a52c7c3efb4eb6ea1c0e202c3bae875aaead38778
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGE_DECLARED=false
RESULT_CONTRACT_VALID=true
RESULT_CONTRACT_STATUS=NOT_APPLICABLE
RESULT_CONTRACT_ERROR=NONE
COMMAND_RC=0
CHANGES_MADE=false
ROLLBACK_STARTED=false
ROLLBACK_RESTORED=null
MUTATION_OUTCOME=NO_MUTATION
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=7789c8f8591d0be49d972a2812151287eb12642bd99740576b1b9d9a29f84267
SANITIZED_OUTPUT_SHA256=5b17a087bd51a9f4e901cd820e129160e70081636d2da63d902f481305df4f99
OUTPUT_BEGIN
HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z","command_rc":0,"control_plane_error":null,"exact_source_count":30,"index_source_count":1,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_atomic_transaction":true,"reference_source_count":1,"remote_probe_privilege":"sudo-root","report_bytes":193466,"report_sha256":"772f2c3f77d7909de34aa159e0e2c8c2ca5480ff51cbcd4bcb194af9019b43d0","rollback_restored":false,"rollback_started":false,"status":"OK","unit_record_count":16,"v15_declared_report_sha256":"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf","v15_exact_result_identity_confirmed":true,"v15_output_sha256":"c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8","v15_sanitized_report_sha256":"f05595551c37c00d473df29113cf9d597b2d3467ad5a2b50e8803e8579accc82","v15_sanitizer_byte_delta":50,"version":1}
{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z","mutation_outcome":"NO_MUTATION","read_only":true,"remote_exact_map":{"euid":0,"exact_manifest":[{"bytes":3708,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","priority":100,"requested_as":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","uid":0},{"bytes":10353,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","priority":95,"requested_as":"/var/www/homelab-cluster-admin/index.html","sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","uid":0},{"bytes":2832,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","priority":92,"requested_as":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5","uid":0},{"bytes":3471,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","priority":92,"requested_as":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e","uid":0},{"bytes":1433,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","priority":92,"requested_as":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792","uid":0},{"bytes":848,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","priority":92,"requested_as":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972","uid":0},{"error":"RuntimeError: unsafe file: /usr/bin/python3.11","is_dir":false,"is_file":true,"is_symlink":false,"path":"/usr/bin/python3.11","priority":92,"requested_as":"/usr/bin/python3"},{"bytes":2541,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/usr/local/sbin/homelab-stage4i-verify","priority":92,"requested_as":"/usr/local/sbin/homelab-stage4i-verify","sha256":"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d","uid":0},{"bytes":616,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","priority":90,"requested_as":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7","uid":0},{"bytes":231,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","priority":90,"requested_as":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae","uid":0},{"bytes":803,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","priority":90,"requested_as":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea","uid":0},{"bytes":1190,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","priority":90,"requested_as":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771","uid":0},{"bytes":223,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","priority":90,"requested_as":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d","uid":0},{"bytes":133,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b","uid":0},{"bytes":154,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23","uid":0},{"bytes":131,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042","uid":0},{"bytes":149,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b","uid":0},{"bytes":127,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d","uid":0},{"bytes":151,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1","uid":0},{"bytes":138,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e","uid":0},{"bytes":154,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311","uid":0},{"bytes":301,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","uid":0},{"bytes":465,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.service","priority":90,"requested_as":"/etc/systemd/system/homelab-stage4i-verify.service","sha256":"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b","uid":0},{"bytes":224,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.timer","priority":90,"requested_as":"/etc/systemd/system/homelab-stage4i-verify.timer","sha256":"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396","uid":0},{"is_dir":true,"is_file":false,"is_symlink":false,"path":"/opt/cluster-admin-incident-engine","priority":75,"requested_as":"/opt/cluster-admin-incident-engine"},{"bytes":380,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","priority":70,"requested_as":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e","uid":0},{"bytes":217,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","priority":70,"requested_as":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053","uid":0},{"bytes":128,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","priority":70,"requested_as":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac","uid":0},{"bytes":1535,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","priority":70,"requested_as":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b","uid":0},{"bytes":616,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","priority":65,"requested_as":"nearby","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7","uid":0},{"bytes":231,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","priority":65,"requested_as":"nearby","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae","uid":0},{"bytes":803,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","priority":65,"requested_as":"nearby","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea","uid":0},{"bytes":1190,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","priority":65,"requested_as":"nearby","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771","uid":0},{"bytes":223,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","priority":65,"requested_as":"nearby","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d","uid":0},{"bytes":133,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","priority":65,"requested_as":"nearby","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b","uid":0},{"bytes":154,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","priority":65,"requested_as":"nearby","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23","uid":0},{"bytes":131,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","priority":65,"requested_as":"nearby","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042","uid":0},{"bytes":149,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","priority":65,"requested_as":"nearby","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b","uid":0},{"bytes":127,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","priority":65,"requested_as":"nearby","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d","uid":0},{"bytes":151,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","priority":65,"requested_as":"nearby","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1","uid":0},{"bytes":138,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","priority":65,"requested_as":"nearby","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e","uid":0},{"bytes":154,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","priority":65,"requested_as":"nearby","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311","uid":0},{"bytes":301,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","priority":65,"requested_as":"nearby","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","uid":0},{"bytes":465,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.service","priority":65,"requested_as":"nearby","sha256":"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b","uid":0},{"bytes":224,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.timer","priority":65,"requested_as":"nearby","sha256":"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396","uid":0},{"bytes":2832,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","priority":65,"requested_as":"nearby","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5","uid":0},{"bytes":3471,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","priority":65,"requested_as":"nearby","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e","uid":0},{"bytes":1433,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","priority":65,"requested_as":"nearby","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792","uid":0},{"bytes":3708,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","priority":65,"requested_as":"nearby","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","uid":0},{"bytes":848,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","priority":65,"requested_as":"nearby","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972","uid":0},{"bytes":10353,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","priority":65,"requested_as":"nearby","sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","uid":0},{"bytes":89,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","priority":50,"requested_as":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f","uid":0},{"bytes":28599,"gid":994,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6","uid":0},{"bytes":10640,"gid":994,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe","uid":0},{"bytes":1857,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670","uid":0},{"bytes":2355,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588","uid":0},{"bytes":1342,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c","uid":0},{"bytes":922,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3","uid":0},{"bytes":8444,"gid":994,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","priority":50,"requested_as":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a","uid":0},{"bytes":131,"gid":994,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","priority":50,"requested_as":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"5348849c615d0258d7812fdcf660611f911b97753bbab2e0fd87e5b1c2a98cf8","uid":999},{"bytes":259,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"275b09782748c2b8a1b762b6390a84df202e478ece7e8f377788569b2cc5b027","uid":0},{"bytes":168,"gid":994,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"5b2bc5ae640f3b7b42000bae6b5b63d76ba00a8f55f4b6cc1cc9f80871373653","uid":999},{"bytes":189,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941","uid":0},{"bytes":141,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6","uid":0},{"bytes":199,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680","uid":0},{"bytes":272,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"49dbf42b944b78c7394d299a08400d24a92b7532fe8d9774c686fe6a59972e15","uid":0},{"bytes":134,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"6353b738b98d4d038684bc753014410f96cf0ace91227dad461b4a2c69ee3c29","uid":0},{"bytes":214,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"02916ca8a08ee305709d7fa7935209a6321b0e25fb105e0f2d3e242ed66fb0f0","uid":0},{"bytes":214,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","priority":50,"requested_as":"/var/www/homelab-cluster-admin/status.txt","sha256":"02916ca8a08ee305709d7fa7935209a6321b0e25fb105e0f2d3e242ed66fb0f0","uid":0}],"exact_sources":[{"bytes":3708,"line_count":79,"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","priority":100,"sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":5,"text":"mkdir -p \"$WEB\""},{"line":6,"text":"BAD=0"},{"line":7,"text":""},{"line":8,"text":"line_or_missing() {"},{"line":9,"text":"  f=\"$1\""},{"line":10,"text":"  if [ -s \"$f\" ]; then head -n1 \"$f\"; else echo \"STATUS=WARN TYPE=missing FILE=$f\"; fi"},{"line":11,"text":"}"},{"line":12,"text":""},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":20,"text":"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":21,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":22,"text":""},{"line":23,"text":"for line in \"$SELF\" \"$OBSERVER\" \"$FULL\" \"$BACKUP\"; do"},{"line":24,"text":"  echo \"$line\" | grep -q '^STATUS=OK' || BAD=$((BAD+1))"},{"line":25,"text":"done"},{"line":26,"text":"echo \"$FULL\" | grep -q 'BAD=0' || BAD=$((BAD+1))"},{"line":27,"text":"echo \"$FULL\" | grep -q 'WARN=0' || BAD=$((BAD+1))"},{"line":28,"text":"echo \"$BACKUP\" | grep -q 'BAD=0' || BAD=$((BAD+1))"},{"line":29,"text":"echo \"$BACKUP\" | grep -q 'MAIL_CLOUD_2COPY_VERIFY_OK' || BAD=$((BAD+1))"},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":31,"text":"[ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":32,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":33,"text":""},{"line":34,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; COLOR=\"#0a7f38\"; BADGE=\"OK\"; else STATUS=WARN; COLOR=\"#b26a00\"; BADGE=\"REVIEW\"; fi"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":37,"text":"echo \"$STATUS_LINE\" > \"$WEB/status.txt\""},{"line":38,"text":""},{"line":39,"text":"cat > \"$WEB/index.html\" <<HTML"},{"line":40,"text":"<!doctype html>"},{"line":41,"text":"<html lang=\"ru\">"},{"line":42,"text":"<head>"},{"line":43,"text":"<meta charset=\"utf-8\">"},{"line":44,"text":"<meta http-equiv=\"refresh\" content=\"60\">"},{"line":45,"text":"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":47,"text":"<style>"},{"line":48,"text":"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}"},{"line":49,"text":".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}"},{"line":50,"text":"h1{margin:0 0 8px}"},{"line":51,"text":".badge{display:inline-block;background:$COLOR;color:white;padding:8px 14px;border-radius:999px;font-weight:700}"},{"line":52,"text":"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}"},{"line":53,"text":".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}"},{"line":54,"text":".small{color:#aab2c0}"},{"line":55,"text":"</style>"},{"line":56,"text":"</head>"},{"line":57,"text":"<body>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":59,"text":"<div class=\"small\">generated UTC: $TS · VM180 · [PRIVATE_IP] · refresh 60s</div>"},{"line":60,"text":"<p><span class=\"badge\">$BADGE</span></p>"},{"line":61,"text":""},{"line":62,"text":"<div class=\"grid\">"},{"line":63,"text":"<div class=\"card\"><h2>Self</h2><pre>$SELF"},{"line":64,"text":"FAILED_UNITS=$FAILED"},{"line":65,"text":"QGA=$QGA</pre></div>"},{"line":66,"text":"<div class=\"card\"><h2>Observer</h2><pre>$OBSERVER</pre></div>"},{"line":67,"text":"<div class=\"card\"><h2>Full observer</h2><pre>$FULL</pre></div>"},{"line":68,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>$BACKUP</pre></div>"},{"line":69,"text":"</div>"},{"line":70,"text":""},{"line":71,"text":"<div class=\"card\"><h2>Overall</h2><pre>$OVERALL</pre></div>"},{"line":72,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>$FINAL</pre></div>"},{"line":73,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>$STRICT</pre></div>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":75,"text":"</body>"},{"line":76,"text":"</html>"},{"line":77,"text":"HTML"},{"line":78,"text":""},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}]},{"bytes":10353,"line_count":80,"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","priority":95,"sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","source":[{"line":1,"text":"<!doctype html>"},{"line":2,"text":"<html lang=\"ru\">"},{"line":3,"text":"<head>"},{"line":4,"text":"<meta charset=\"utf-8\">"},{"line":5,"text":"<meta http-equiv=\"refresh\" content=\"60\">"},{"line":6,"text":"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"},{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":8,"text":"<style>"},{"line":9,"text":"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}"},{"line":10,"text":".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}"},{"line":11,"text":"h1{margin:0 0 8px}"},{"line":12,"text":".badge{display:inline-block;background:#b26a00;color:white;padding:8px 14px;border-radius:999px;font-weight:700}"},{"line":13,"text":"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}"},{"line":14,"text":".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}"},{"line":15,"text":".small{color:#aab2c0}"},{"line":16,"text":"</style>"},{"line":17,"text":"</head>"},{"line":18,"text":"<body>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":20,"text":"<div class=\"small\">generated UTC: 2026-08-05T13:20:48Z · VM180 · [PRIVATE_IP] · refresh 60s</div>"},{"line":21,"text":"<p><span class=\"badge\">REVIEW</span></p>"},{"line":22,"text":""},{"line":23,"text":"<div class=\"grid\">"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":25,"text":"FAILED_UNITS=1"},{"line":26,"text":"QGA=active</pre></div>"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":30,"text":"</div>"},{"line":31,"text":""},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":36,"text":"generated_utc=2026-08-05T13:19:08Z"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":38,"text":""},{"line":39,"text":"## Restricted probes"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":44,"text":""},{"line":45,"text":"## Service map checks"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":47,"text":"OK severity=critical kind=tcp name=pve01_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":48,"text":"OK severity=critical kind=tcp name=pve01_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0"},{"line":49,"text":"OK severity=critical kind=tcp name=pve01_health target=[PRIVATE_IP] value=9101 tcp_9101_rc=0"},{"line":50,"text":"OK severity=critical kind=tcp name=pve02_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":51,"text":"OK severity=critical kind=tcp name=pve02_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0"},{"line":52,"text":"OK severity=critical kind=tcp name=pve03_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":53,"text":"OK severity=critical kind=tcp name=pve03_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0"},{"line":54,"text":"OK severity=critical kind=tcp name=dns1_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0"},{"line":55,"text":"OK severity=critical kind=tcp name=dns2_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0"},{"line":56,"text":"OK severity=critical kind=tcp name=edge_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":58,"text":"OK severity=critical kind=ping name=vm130_edge_vm_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":59,"text":"OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":60,"text":"OK severity=critical kind=ping name=vm150_nextcloud_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":61,"text":"OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":62,"text":"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1"},{"line":63,"text":"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":64,"text":"OK severity=critical kind=ping name=vm160_forum_prod_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":65,"text":"OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":66,"text":"OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=0"},{"line":67,"text":"INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":68,"text":"OK severity=critical kind=ping name=vm170_core_apps_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":69,"text":"OK severity=critical kind=tcp name=vm170_core_apps_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":70,"text":"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1"},{"line":71,"text":"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":72,"text":"OK severity=critical kind=ping name=vm171_monitoring_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":73,"text":"OK severity=critical kind=tcp name=vm171_monitoring_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":74,"text":"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1"},{"line":75,"text":"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":76,"text":""},{"line":77,"text":"## Result"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":79,"text":"</body>"},{"line":80,"text":"</html>"}]},{"bytes":2832,"line_count":96,"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","priority":92,"sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":4,"text":"exec 9>\"$LOCK\""},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":6,"text":""},{"line":7,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":10,"text":"REPORT=\"$ROOT/latest.md\""},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":12,"text":"TMP=\"$REPORT.tmp\""},{"line":13,"text":"mkdir -p \"$ROOT\" \"$HEALTH_DIR\""},{"line":14,"text":"BAD=0"},{"line":15,"text":"WARN=0"},{"line":16,"text":""},{"line":17,"text":"check_ping() {"},{"line":18,"text":"  name=\"$1\"; ip=\"$2\""},{"line":19,"text":"  ping -c1 -W1 \"$ip\" >/dev/null 2>&1"},{"line":20,"text":"  rc=$?"},{"line":21,"text":"  echo \"$name ping rc=$rc\" >> \"$TMP\""},{"line":22,"text":"  [ \"$rc\" = 0 ] || BAD=$((BAD+1))"},{"line":23,"text":"}"},{"line":24,"text":""},{"line":25,"text":"check_tcp() {"},{"line":26,"text":"  name=\"$1\"; ip=\"$2\"; port=\"$3\""},{"line":27,"text":"  timeout 2 bash -c \"</dev/tcp/$ip/$port\" >/dev/null 2>&1"},{"line":28,"text":"  rc=$?"},{"line":29,"text":"  echo \"$name tcp/$port rc=$rc\" >> \"$TMP\""},{"line":30,"text":"  [ \"$rc\" = 0 ] || BAD=$((BAD+1))"},{"line":31,"text":"}"},{"line":32,"text":""},{"line":33,"text":"check_http_status_ok() {"},{"line":34,"text":"  name=\"$1\"; url=\"$2\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":36,"text":"  rc=$?"},{"line":37,"text":"  echo \"$name http rc=$rc line=$line\" >> \"$TMP\""},{"line":38,"text":"  echo \"$line\" | grep -q \"^STATUS=\""},{"line":39,"text":"  okrc=$?"},{"line":40,"text":"  [ \"$rc\" = 0 ] && [ \"$okrc\" = 0 ] || BAD=$((BAD+1))"},{"line":41,"text":"}"},{"line":42,"text":""},{"line":43,"text":"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":44,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":46,"text":""},{"line":47,"text":"{"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":49,"text":"  echo \"generated_utc=$TS\""},{"line":50,"text":"  echo"},{"line":51,"text":"  echo \"## Self\""},{"line":52,"text":"  echo \"host=$(hostname 2>/dev/null)\""},{"line":53,"text":"  echo \"failed_units=$FAILED\""},{"line":54,"text":"  echo \"qga=$QGA\""},{"line":55,"text":"  echo \"self_check=$SELF_LINE\""},{"line":56,"text":"  echo"},{"line":57,"text":"  echo \"## Network checks\""},{"line":58,"text":"} > \"$TMP\""},{"line":59,"text":""},{"line":60,"text":"[ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":61,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":62,"text":"echo \"$SELF_LINE\" | grep -q \"^STATUS=OK\" || BAD=$((BAD+1))"},{"line":63,"text":""},{"line":64,"text":"check_ping pve01 [PRIVATE_IP]"},{"line":65,"text":"check_tcp pve01_ssh [PRIVATE_IP] 22"},{"line":66,"text":"check_tcp pve01_proxmox [PRIVATE_IP] 8006"},{"line":67,"text":"check_tcp pve01_health [PRIVATE_IP] 9101"},{"line":68,"text":""},{"line":69,"text":"check_ping pve02 [PRIVATE_IP]"},{"line":70,"text":"check_tcp pve02_ssh [PRIVATE_IP] 22"},{"line":71,"text":"check_tcp pve02_proxmox [PRIVATE_IP] 8006"},{"line":72,"text":""},{"line":73,"text":"check_ping pve03 [PRIVATE_IP]"},{"line":74,"text":"check_tcp pve03_ssh [PRIVATE_IP] 22"},{"line":75,"text":"check_tcp pve03_proxmox [PRIVATE_IP] 8006"},{"line":76,"text":""},{"line":77,"text":"check_ping edge [PRIVATE_IP]"},{"line":78,"text":"check_tcp edge_ssh [PRIVATE_IP] 22"},{"line":79,"text":""},{"line":80,"text":"check_ping dns1 [PRIVATE_IP]"},{"line":81,"text":"check_tcp dns1_tcp53 [PRIVATE_IP] 53"},{"line":82,"text":"check_ping dns2 [PRIVATE_IP]"},{"line":83,"text":"check_tcp dns2_tcp53 [PRIVATE_IP] 53"},{"line":84,"text":""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":86,"text":""},{"line":87,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""},{"line":89,"text":"echo \"$LINE\" | tee \"$HEALTH\" >/dev/null"},{"line":90,"text":"{"},{"line":91,"text":"  echo"},{"line":92,"text":"  echo \"## Result\""},{"line":93,"text":"  echo \"$LINE\""},{"line":94,"text":"} >> \"$TMP\""},{"line":95,"text":"mv \"$TMP\" \"$REPORT\""},{"line":96,"text":"cat \"$HEALTH\""}]},{"bytes":3471,"line_count":113,"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","priority":92,"sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":7,"text":"REPORT=\"$ROOT/full-observer-latest.md\""},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":10,"text":"mkdir -p \"$ROOT\" \"$HEALTH_DIR\""},{"line":11,"text":"exec 9>\"$LOCK\""},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":13,"text":""},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":15,"text":"BAD=0"},{"line":16,"text":"WARN=0"},{"line":17,"text":"INFO_OFF=0"},{"line":18,"text":"TOTAL=0"},{"line":19,"text":"CRIT_FAIL=0"},{"line":20,"text":"OPT_OFF=0"},{"line":21,"text":"PROBE_OK=0"},{"line":22,"text":"PROBE_BAD=0"},{"line":23,"text":""},{"line":24,"text":"TMP=\"$REPORT.tmp\""},{"line":25,"text":"{"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":27,"text":"  echo \"generated_utc=$TS\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":29,"text":"  echo"},{"line":30,"text":"  echo \"## Restricted probes\""},{"line":31,"text":"} > \"$TMP\""},{"line":32,"text":""},{"line":33,"text":"for t in \"root@[PRIVATE_IP] pve01\" \"root@[PRIVATE_IP] pve02\" \"root@[PRIVATE_IP] pve03\" \"debian@[PRIVATE_IP] edge\"; do"},{"line":34,"text":"  set -- $t"},{"line":35,"text":"  target=\"$1\"; name=\"$2\""},{"line":36,"text":"  out=$(ssh -n -i \"$K\" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 \"$target\" probe 2>/tmp/full-probe-$name.err)"},{"line":37,"text":"  rc=$?"},{"line":38,"text":"  echo \"$name ssh_rc=$rc $out\" >> \"$TMP\""},{"line":39,"text":"  echo \"$out\" | grep -q '^STATUS=OK'"},{"line":40,"text":"  ok=$?"},{"line":41,"text":"  if [ \"$rc\" = 0 ] && [ \"$ok\" = 0 ]; then PROBE_OK=$((PROBE_OK+1)); else PROBE_BAD=$((PROBE_BAD+1)); BAD=$((BAD+1)); fi"},{"line":42,"text":"done"},{"line":43,"text":""},{"line":44,"text":"{"},{"line":45,"text":"  echo"},{"line":46,"text":"  echo \"## Service map checks\""},{"line":47,"text":"} >> \"$TMP\""},{"line":48,"text":""},{"line":49,"text":"while IFS=\"$(printf '\\t')\" read -r severity kind name target value; do"},{"line":50,"text":"  [ -n \"$severity\" ] || continue"},{"line":51,"text":"  TOTAL=$((TOTAL+1))"},{"line":52,"text":"  rc=0"},{"line":53,"text":"  detail=\"\""},{"line":54,"text":"  case \"$kind\" in"},{"line":55,"text":"    ping)"},{"line":56,"text":"      ping -c1 -W1 \"$target\" >/dev/null 2>&1"},{"line":57,"text":"      rc=$?"},{"line":58,"text":"      detail=\"ping_rc=$rc\""},{"line":59,"text":"      ;;"},{"line":60,"text":"    tcp)"},{"line":61,"text":"      timeout 2 bash -c \"</dev/tcp/$target/$value\" >/dev/null 2>&1"},{"line":62,"text":"      rc=$?"},{"line":63,"text":"      detail=\"tcp_${value}_rc=$rc\""},{"line":64,"text":"      ;;"},{"line":65,"text":"    http_status_ok)"},{"line":66,"text":"      line=$(curl -fsS --max-time 5 \"$value\" 2>/tmp/full-http-$name.err | head -n1)"},{"line":67,"text":"      curlrc=$?"},{"line":68,"text":"      echo \"$line\" | grep -q '^STATUS='"},{"line":69,"text":"      okrc=$?"},{"line":70,"text":"      [ \"$curlrc\" = 0 ] && [ \"$okrc\" = 0 ]"},{"line":71,"text":"      rc=$?"},{"line":72,"text":"      detail=\"http_rc=$curlrc ok_rc=$okrc line=$line\""},{"line":73,"text":"      ;;"},{"line":74,"text":"    missing)"},{"line":75,"text":"      rc=1"},{"line":76,"text":"      detail=\"missing_target=$target value=$value\""},{"line":77,"text":"      ;;"},{"line":78,"text":"    *)"},{"line":79,"text":"      rc=1"},{"line":80,"text":"      detail=\"unknown_kind=$kind\""},{"line":81,"text":"      ;;"},{"line":82,"text":"  esac"},{"line":83,"text":""},{"line":84,"text":"  if [ \"$rc\" = 0 ]; then"},{"line":85,"text":"    echo \"OK severity=$severity kind=$kind name=$name target=$target value=$value $detail\" >> \"$TMP\""},{"line":86,"text":"  else"},{"line":87,"text":"    if [ \"$severity\" = critical ]; then"},{"line":88,"text":"      echo \"FAIL severity=critical kind=$kind name=$name target=$target value=$value $detail\" >> \"$TMP\""},{"line":89,"text":"      CRIT_FAIL=$((CRIT_FAIL+1))"},{"line":90,"text":"      BAD=$((BAD+1))"},{"line":91,"text":"    else"},{"line":92,"text":"      echo \"INFO_OFF severity=optional kind=$kind name=$name target=$target value=$value $detail\" >> \"$TMP\""},{"line":93,"text":"      OPT_OFF=$((OPT_OFF+1))"},{"line":94,"text":"      INFO_OFF=$((INFO_OFF+1))"},{"line":95,"text":"    fi"},{"line":96,"text":"  fi"},{"line":97,"text":"done < \"$MAP\""},{"line":98,"text":""},{"line":99,"text":"SELF_FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":100,"text":"[ \"$SELF_FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":101,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":102,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":103,"text":""},{"line":104,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi"},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""},{"line":106,"text":"echo \"$LINE\" | tee \"$HEALTH\" >/dev/null"},{"line":107,"text":"{"},{"line":108,"text":"  echo"},{"line":109,"text":"  echo \"## Result\""},{"line":110,"text":"  echo \"$LINE\""},{"line":111,"text":"} >> \"$TMP\""},{"line":112,"text":"mv \"$TMP\" \"$REPORT\""},{"line":113,"text":"cat \"$HEALTH\""}]},{"bytes":1433,"line_count":27,"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","priority":92,"sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set -Eeuo pipefail"},{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"},{"line":6,"text":"ORIGINAL_RC=0"},{"line":7,"text":"\"$ORIGINAL\" || ORIGINAL_RC=$?"},{"line":8,"text":"[ -s \"$HEALTH\" ] || exit \"$ORIGINAL_RC\""},{"line":9,"text":"line=\"$(sed -n \"1p\" \"$HEALTH\" 2>/dev/null || true)\""},{"line":10,"text":"field(){ local key=<REDACTED>"},{"line":11,"text":"self_failed=\"$(field SELF_FAILED_UNITS || true)\""},{"line":12,"text":"qga_state=\"$(field QGA || true)\""},{"line":13,"text":"backup_state=\"$(field BACKUP || true)\""},{"line":14,"text":"if [ \"$ORIGINAL_RC\" -eq 0 ] && [ \"$self_failed\" = \"0\" ] && [ \"$qga_state\" = \"active\" ] && [ \"$backup_state\" = \"MAIL_CLOUD_2COPY_VERIFY_OK\" ]; then"},{"line":15,"text":"  normalized=\"$(printf \"%s\\n\" \"$line\" | sed -E \"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\")\""},{"line":16,"text":"  printf \"%s\\n\" \"$normalized\" > \"$HEALTH\""},{"line":17,"text":"  if [ -f \"$STATUS_FILE\" ]; then"},{"line":18,"text":"    status_line=\"$(sed -n \"1p\" \"$STATUS_FILE\" 2>/dev/null || true)\""},{"line":19,"text":"    status_normalized=\"$(printf \"%s\\n\" \"$status_line\" | sed -E \"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\")\""},{"line":20,"text":"    tmp=\"${STATUS_FILE}.tmp.$$\""},{"line":21,"text":"    { printf \"%s\\n\" \"$status_normalized\"; tail -n +2 \"$STATUS_FILE\" 2>/dev/null || true; } > \"$tmp\""},{"line":22,"text":"    chown --reference=\"$STATUS_FILE\" \"$tmp\" 2>/dev/null || true"},{"line":23,"text":"    chmod --reference=\"$STATUS_FILE\" \"$tmp\" 2>/dev/null || true"},{"line":24,"text":"    mv \"$tmp\" \"$STATUS_FILE\""},{"line":25,"text":"  fi"},{"line":26,"text":"fi"},{"line":27,"text":"exit \"$ORIGINAL_RC\""}]},{"bytes":848,"line_count":17,"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","priority":92,"sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":4,"text":"BAD=0"},{"line":5,"text":"HOST=$(hostname 2>/dev/null || echo unknown)"},{"line":6,"text":"IP_OK=0"},{"line":7,"text":"ip -4 -br a | grep -q \"[PRIVATE_IP]\" && IP_OK=1 || BAD=$((BAD+1))"},{"line":8,"text":"DNS_OK=0"},{"line":9,"text":"getent hosts pve01 >/dev/null 2>&1 || getent hosts gram1.ru >/dev/null 2>&1"},{"line":10,"text":"[ \"$?\" = 0 ] && DNS_OK=1 || BAD=$((BAD+1))"},{"line":11,"text":"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":12,"text":"[ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":13,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":14,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":15,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi"},{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}]},{"bytes":2541,"line_count":105,"mode":"0o750","path":"/usr/local/sbin/homelab-stage4i-verify","priority":92,"sha256":"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set -euo pipefail"},{"line":3,"text":"export LC_ALL=C"},{"line":4,"text":""},{"line":5,"text":"MARKER=\"CR0019_STAGE4I_IMMUTABLE_V1\""},{"line":6,"text":"STATE_DIR=\"/var/lib/homelab-cluster-admin/stage4i-verification\""},{"line":7,"text":"REPORT=\"$STATE_DIR/latest.env\""},{"line":8,"text":"TMP=\"$(mktemp)\""},{"line":9,"text":""},{"line":10,"text":"cleanup() {"},{"line":11,"text":"    rm -f \"$TMP\""},{"line":12,"text":"}"},{"line":13,"text":"trap cleanup EXIT"},{"line":14,"text":""},{"line":15,"text":"mapfile -t ACTIVATION_FILES < <("},{"line":16,"text":"    find \\"},{"line":17,"text":"        /var/lib/homelab-cluster-admin \\"},{"line":18,"text":"        /opt \\"},{"line":19,"text":"        /srv \\"},{"line":20,"text":"        /usr/local/share \\"},{"line":21,"text":"        -type f \\"},{"line":22,"text":"        -path '*/.stage4i-control/activation.env' \\"},{"line":23,"text":"        -print 2>/dev/null |"},{"line":24,"text":"    sort -u"},{"line":25,"text":")"},{"line":26,"text":""},{"line":27,"text":"if [ \"${#ACTIVATION_FILES[@]}\" -ne 1 ]; then"},{"line":28,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=FAILED\""},{"line":29,"text":"    echo \"REASON=ACTIVATION_FILE_COUNT_${#ACTIVATION_FILES[@]}\""},{"line":30,"text":"    exit 50"},{"line":31,"text":"fi"},{"line":32,"text":""},{"line":33,"text":"ACTIVATION_FILE=\"${ACTIVATION_FILES[0]}\""},{"line":34,"text":"CONTROL_DIR=\"$(dirname \"$ACTIVATION_FILE\")\""},{"line":35,"text":"ACTIVE_LINK=\"$CONTROL_DIR/active\""},{"line":36,"text":"ROLLBACK_LINK=\"$CONTROL_DIR/rollback\""},{"line":37,"text":""},{"line":38,"text":"# shellcheck disable=SC1090"},{"line":39,"text":"source \"$ACTIVATION_FILE\""},{"line":40,"text":""},{"line":41,"text":"test \"${STAGE4I_ACTIVATION_STATUS:-}\" = \"ACTIVE\""},{"line":42,"text":"test -L \"$ACTIVE_LINK\""},{"line":43,"text":"test -L \"$ROLLBACK_LINK\""},{"line":44,"text":""},{"line":45,"text":"ACTIVE_RESOLVED=\"$(readlink -f \"$ACTIVE_LINK\")\""},{"line":46,"text":"ROLLBACK_RESOLVED=\"$(readlink -f \"$ROLLBACK_LINK\")\""},{"line":47,"text":""},{"line":48,"text":"test \"$ACTIVE_RESOLVED\" = \"${ACTIVE_PATH:-}\""},{"line":49,"text":"test \"$ROLLBACK_RESOLVED\" = \"${ROLLBACK_PATH:-}\""},{"line":50,"text":"test -d \"$ACTIVE_RESOLVED\""},{"line":51,"text":"test -d \"$ROLLBACK_RESOLVED\""},{"line":52,"text":""},{"line":53,"text":"META=\"$ACTIVE_RESOLVED/IMMUTABLE.env\""},{"line":54,"text":"MANIFEST=\"$ACTIVE_RESOLVED/MANIFEST.sha256\""},{"line":55,"text":""},{"line":56,"text":"test -f \"$META\""},{"line":57,"text":"test -f \"$MANIFEST\""},{"line":58,"text":"grep -q \"^BUILD_MARKER=$MARKER$\" \"$META\""},{"line":59,"text":"grep -q '^TASK_STAGE=4I$' \"$META\""},{"line":60,"text":"grep -q '^TASK_STATUS=IMMUTABLE$' \"$META\""},{"line":61,"text":""},{"line":62,"text":"("},{"line":63,"text":"    cd \"$ACTIVE_RESOLVED\""},{"line":64,"text":"    sha256sum -c MANIFEST.sha256 >/dev/null"},{"line":65,"text":")"},{"line":66,"text":""},{"line":67,"text":"if find \"$ACTIVE_RESOLVED\" \\"},{"line":68,"text":"    \\( ! -user root -o ! -group root \\) \\"},{"line":69,"text":"    -print -quit |"},{"line":70,"text":"    grep -q ."},{"line":71,"text":"then"},{"line":72,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=FAILED\""},{"line":73,"text":"    echo \"REASON=INVALID_OWNER\""},{"line":74,"text":"    exit 51"},{"line":75,"text":"fi"},{"line":76,"text":""},{"line":77,"text":"if find \"$ACTIVE_RESOLVED\" \\"},{"line":78,"text":"    -perm /0222 \\"},{"line":79,"text":"    -print -quit |"},{"line":80,"text":"    grep -q ."},{"line":81,"text":"then"},{"line":82,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=FAILED\""},{"line":83,"text":"    echo \"REASON=WRITABLE_IMMUTABLE_CONTENT\""},{"line":84,"text":"    exit 52"},{"line":85,"text":"fi"},{"line":86,"text":""},{"line":87,"text":"install -d -m 0750 -o root -g root \"$STATE_DIR\""},{"line":88,"text":""},{"line":89,"text":"{"},{"line":90,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=SUCCESS\""},{"line":91,"text":"    echo \"VERIFIED_AT_UTC=$(date -u +%Y-%m-%dT%H:%M:%SZ)\""},{"line":92,"text":"    echo \"ACTIVE_VERSION=${ACTIVE_VERSION:-UNKNOWN}\""},{"line":93,"text":"    echo \"ACTIVE_PATH=$ACTIVE_RESOLVED\""},{"line":94,"text":"    echo \"ROLLBACK_VERSION=${ROLLBACK_VERSION:-UNKNOWN}\""},{"line":95,"text":"    echo \"ROLLBACK_PATH=$ROLLBACK_RESOLVED\""},{"line":96,"text":"    echo \"MANIFEST_SHA256=$(sha256sum \"$MANIFEST\" | awk '{print $1}')\""},{"line":97,"text":"    echo \"VERIFIED_ON_HOST=$(hostname -f 2>/dev/null || hostname)\""},{"line":98,"text":"} >\"$TMP\""},{"line":99,"text":""},{"line":100,"text":"chown root:root \"$TMP\""},{"line":101,"text":"chmod 0440 \"$TMP\""},{"line":102,"text":"mv -f \"$TMP\" \"$REPORT\""},{"line":103,"text":"trap - EXIT"},{"line":104,"text":""},{"line":105,"text":"cat \"$REPORT\""}]},{"bytes":616,"line_count":19,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","priority":90,"sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":"Requires=postgresql.service"},{"line":6,"text":""},{"line":7,"text":"[Service]"},{"line":8,"text":"Type=oneshot"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":13,"text":"UMask=0027"},{"line":14,"text":"NoNewPrivileges=true"},{"line":15,"text":"PrivateTmp=true"},{"line":16,"text":"ProtectSystem=strict"},{"line":17,"text":"ProtectHome=true"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}]},{"bytes":231,"line_count":12,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","priority":90,"sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=30s"},{"line":6,"text":"OnUnitActiveSec=60s"},{"line":7,"text":"AccuracySec=10s"},{"line":8,"text":"Persistent=true"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"},{"line":10,"text":""},{"line":11,"text":"[Install]"},{"line":12,"text":"WantedBy=timers.target"}]},{"bytes":803,"line_count":28,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","priority":90,"sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":3,"text":"After=network-online.target postgresql.service"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":"Requires=postgresql.service"},{"line":6,"text":""},{"line":7,"text":"[Service]"},{"line":8,"text":"Type=simple"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":13,"text":"ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1"},{"line":14,"text":"Restart=on-failure"},{"line":15,"text":"RestartSec=5"},{"line":16,"text":"UMask=0027"},{"line":17,"text":"NoNewPrivileges=true"},{"line":18,"text":"PrivateTmp=true"},{"line":19,"text":"ProtectSystem=strict"},{"line":20,"text":"ProtectHome=true"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"},{"line":23,"text":"RestrictSUIDSGID=true"},{"line":24,"text":"LockPersonality=true"},{"line":25,"text":"RestrictRealtime=true"},{"line":26,"text":""},{"line":27,"text":"[Install]"},{"line":28,"text":"WantedBy=multi-user.target"}]},{"bytes":1190,"line_count":42,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","priority":90,"sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":3,"text":"After=network-online.target"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":""},{"line":6,"text":"[Service]"},{"line":7,"text":"Type=oneshot"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":12,"text":"UMask=0077"},{"line":13,"text":"NoNewPrivileges=yes"},{"line":14,"text":"PrivateTmp=yes"},{"line":15,"text":"PrivateDevices=yes"},{"line":16,"text":"ProtectSystem=strict"},{"line":17,"text":"ProtectHome=yes"},{"line":18,"text":"ProtectKernelTunables=yes"},{"line":19,"text":"ProtectKernelModules=yes"},{"line":20,"text":"ProtectKernelLogs=yes"},{"line":21,"text":"ProtectControlGroups=yes"},{"line":22,"text":"ProtectClock=yes"},{"line":23,"text":"ProtectHostname=yes"},{"line":24,"text":"LockPersonality=yes"},{"line":25,"text":"MemoryDenyWriteExecute=yes"},{"line":26,"text":"RestrictSUIDSGID=yes"},{"line":27,"text":"RestrictRealtime=yes"},{"line":28,"text":"RemoveIPC=yes"},{"line":29,"text":"CapabilityBoundingSet="},{"line":30,"text":"AmbientCapabilities="},{"line":31,"text":"RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6"},{"line":32,"text":"IPAddressDeny=any"},{"line":33,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":34,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":35,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":36,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":40,"text":"StateDirectoryMode=0750"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"},{"line":42,"text":"TimeoutStartSec=70"}]},{"bytes":223,"line_count":12,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","priority":90,"sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=2min"},{"line":6,"text":"OnUnitActiveSec=60s"},{"line":7,"text":"RandomizedDelaySec=5s"},{"line":8,"text":"Persistent=true"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"},{"line":10,"text":""},{"line":11,"text":"[Install]"},{"line":12,"text":"WantedBy=timers.target"}]},{"bytes":133,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","priority":90,"sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}]},{"bytes":154,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","priority":90,"sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run homelab cluster-admin full observer"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=5min"},{"line":6,"text":"OnUnitActiveSec=5min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]},{"bytes":131,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","priority":90,"sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}]},{"bytes":149,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","priority":90,"sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run homelab cluster-admin observer"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=3min"},{"line":6,"text":"OnUnitActiveSec=5min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]},{"bytes":127,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","priority":90,"sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}]},{"bytes":151,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","priority":90,"sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run homelab cluster-admin self check"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=2min"},{"line":6,"text":"OnUnitActiveSec=5min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]},{"bytes":138,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","priority":90,"sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}]},{"bytes":154,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","priority":90,"sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=1min"},{"line":6,"text":"OnUnitActiveSec=1min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]},{"bytes":301,"line_count":13,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","priority":90,"sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":3,"text":"After=network-online.target"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":""},{"line":6,"text":"[Service]"},{"line":7,"text":"Type=simple"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"},{"line":9,"text":"Restart=always"},{"line":10,"text":"RestartSec=5"},{"line":11,"text":""},{"line":12,"text":"[Install]"},{"line":13,"text":"WantedBy=multi-user.target"}]},{"bytes":465,"line_count":22,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.service","priority":90,"sha256":"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Verify active immutable Stage 4I task version"},{"line":3,"text":"After=local-fs.target"},{"line":4,"text":""},{"line":5,"text":"[Service]"},{"line":6,"text":"Type=oneshot"},{"line":7,"text":"User=root"},{"line":8,"text":"Group=root"},{"line":9,"text":"ExecStart=/usr/local/sbin/homelab-stage4i-verify"},{"line":10,"text":"NoNewPrivileges=true"},{"line":11,"text":"PrivateDevices=true"},{"line":12,"text":"ProtectClock=true"},{"line":13,"text":"ProtectControlGroups=true"},{"line":14,"text":"ProtectHome=true"},{"line":15,"text":"ProtectKernelLogs=true"},{"line":16,"text":"ProtectKernelModules=true"},{"line":17,"text":"ProtectKernelTunables=true"},{"line":18,"text":"RestrictNamespaces=true"},{"line":19,"text":"RestrictRealtime=true"},{"line":20,"text":"LockPersonality=true"},{"line":21,"text":"MemoryDenyWriteExecute=true"},{"line":22,"text":"UMask=0027"}]},{"bytes":224,"line_count":12,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.timer","priority":90,"sha256":"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Regularly verify active immutable Stage 4I task version"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=2min"},{"line":6,"text":"OnUnitActiveSec=15min"},{"line":7,"text":"AccuracySec=1min"},{"line":8,"text":"Persistent=true"},{"line":9,"text":"Unit=homelab-stage4i-verify.service"},{"line":10,"text":""},{"line":11,"text":"[Install]"},{"line":12,"text":"WantedBy=timers.target"}]},{"bytes":380,"line_count":1,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","priority":70,"sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e","source":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}]},{"bytes":217,"line_count":1,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","priority":70,"sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053","source":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}]},{"bytes":128,"line_count":1,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","priority":70,"sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac","source":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md"}]},{"bytes":1535,"line_count":1,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","priority":70,"sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b","source":[{"line":1,"text":"STATUS=<REDACTED>"}]},{"bytes":89,"line_count":3,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","priority":50,"sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f","source":[{"line":1,"text":"[Service]"},{"line":2,"text":"ExecStart="},{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}]},{"bytes":28599,"line_count":484,"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","priority":50,"sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6","source":[{"line":1,"text":"<REDACTED_SECRET_LINE>"},{"line":2,"text":"from collections import defaultdict"},{"line":3,"text":"from contextlib import contextmanager"},{"line":4,"text":"from datetime import datetime"},{"line":5,"text":"from urllib.parse import parse_qs"},{"line":6,"text":"import psycopg2"},{"line":7,"text":"from psycopg2.extras import RealDictCursor"},{"line":8,"text":"from fastapi import FastAPI, HTTPException, Request"},{"line":9,"text":"from fastapi.responses import HTMLResponse, RedirectResponse"},{"line":10,"text":"from starlette.middleware.trustedhost import TrustedHostMiddleware"},{"line":11,"text":""},{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE =<REDACTED>"},{"line":14,"text":"COOKIE =<REDACTED>"},{"line":15,"text":"failures, locks = defaultdict(list), {}"},{"line":16,"text":"app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)"},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":18,"text":""},{"line":19,"text":"def db():"},{"line":20,"text":"    return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":21,"text":""},{"line":22,"text":"def enc(value):"},{"line":23,"text":"    return base64.urlsafe_b64encode(value).rstrip(b\"=\").decode()"},{"line":24,"text":""},{"line":25,"text":"def dec(value):"},{"line":26,"text":"    return base64.urlsafe_b64decode(value + \"=\" * (-len(value) % 4))"},{"line":27,"text":""},{"line":28,"text":"def sign(value):"},{"line":29,"text":"    return enc(hmac.new(KEY, value.encode(), hashlib.sha256).digest())"},{"line":30,"text":""},{"line":31,"text":"def ip(request):"},{"line":32,"text":"    peer = request.client.host if request.client else \"unknown\""},{"line":33,"text":"    if peer in {\"[PRIVATE_IP]\",\"127.0.0.1\",\"::1\"}:"},{"line":34,"text":"        return request.headers.get(\"x-forwarded-for\", peer).split(\",\")[0].strip()"},{"line":35,"text":"    return peer"},{"line":36,"text":""},{"line":37,"text":"def login_token(request): <REDACTED>"},{"line":38,"text":"    raw = f\"{int(time.time())}:{ip(request)}\""},{"line":39,"text":"    return raw + \".\" + sign(raw)"},{"line":40,"text":""},{"line":41,"text":"def valid_login_token(request, token): <REDACTED>"},{"line":42,"text":"    try:"},{"line":43,"text":"        raw, signature =<REDACTED>"},{"line":44,"text":"        stamp, address = raw.split(\":\", 1)"},{"line":45,"text":"        return hmac.compare_digest(signature, sign(raw)) and address == ip(request) and 0 <= time.time() - int(stamp) <= 600"},{"line":46,"text":"    except Exception:"},{"line":47,"text":"        return False"},{"line":48,"text":""},{"line":49,"text":"def session_cookie(request): <REDACTED>"},{"line":50,"text":"    data =<REDACTED>"},{"line":51,"text":"    raw = enc(json.dumps(data,separators=(\",\",\":\")).encode())"},{"line":52,"text":"    return raw + \".\" + sign(raw)"},{"line":53,"text":""},{"line":54,"text":"def session(request):"},{"line":55,"text":"    try:"},{"line":56,"text":"        raw, signature =<REDACTED>"},{"line":57,"text":"        data = json.loads(dec(raw))"},{"line":58,"text":"        expected = hashlib.sha256(request.headers.get(\"user-agent\",\"\").encode()).hexdigest()[:24]"},{"line":59,"text":"        return data if hmac.compare_digest(signature,sign(raw)) and data[\"exp\"] >= time.time() and data[\"ua\"] == expected else None"},{"line":60,"text":"    except Exception:"},{"line":61,"text":"        return None"},{"line":62,"text":""},{"line":63,"text":"def password_ok(password): <REDACTED>"},{"line":64,"text":"    try:"},{"line":65,"text":"        scheme, rounds, salt, expected = open(PASSFILE,encoding=\"utf-8\").read().strip().split(\"$\",3)"},{"line":66,"text":"        actual =<REDACTED>"},{"line":67,"text":"        return scheme == \"pbkdf2_sha256\" and hmac.compare_digest(base64.urlsafe_b64encode(actual).decode(),expected)"},{"line":68,"text":"    except Exception:"},{"line":69,"text":"        return False"},{"line":70,"text":""},{"line":71,"text":"def login_page(request, error=\"\"):"},{"line":72,"text":"    token =<REDACTED>"},{"line":73,"text":"    return f\"\"\"<!doctype html><html lang=<REDACTED>"},{"line":74,"text":""},{"line":75,"text":"def status_kind(value):"},{"line":76,"text":"    text = \"UNKNOWN\" if value is None or str(value).strip() == \"\" else str(value).strip().upper()"},{"line":77,"text":"    if text in {\"OK\", \"ACTIVE\", \"RUNNING\", \"UP\", \"ONLINE\", \"HEALTHY\", \"RECOVERED\", \"CLOSED\", \"SUCCESS\", \"ENABLED\", \"READY\"}:"},{"line":78,"text":"        return \"ok\""},{"line":79,"text":"    if text in {\"WARN\", \"WARNING\", \"PENDING\", \"DEGRADED\", \"ATTENTION\", \"STALE\"}:"},{"line":80,"text":"        return \"warn\""},{"line":81,"text":"    if text in {\"OPEN\", \"ERROR\", \"FAILED\", \"DOWN\", \"OFFLINE\", \"CRITICAL\", \"P0\", \"P1\", \"FIRING\", \"UNHEALTHY\"}:"},{"line":82,"text":"        return \"bad\""},{"line":83,"text":"    return \"unknown\""},{"line":84,"text":""},{"line":85,"text":"def status_badge(value):"},{"line":86,"text":"    text = \"UNKNOWN\" if value is None or str(value).strip() == \"\" else str(value)"},{"line":87,"text":"    kind = status_kind(text)"},{"line":88,"text":"    return \"<span class=\\\"state state-\" + kind + \"\\\"><span class=\\\"state-dot\\\"></span>\" + html.escape(text) + \"</span>\""},{"line":89,"text":""},{"line":90,"text":"def summary_card(title, value, state):"},{"line":91,"text":"    return \"<div class=\\\"card\\\"><div class=\\\"card-health\\\">\" + status_badge(state) + \"</div><b>\" + html.escape(str(value)) + \"</b><span>\" + html.escape(title) + \"</span></div>\""},{"line":92,"text":""},{"line":93,"text":"def aggregate_state(rows, keys=(\"status\",), empty_state=\"WARN\"):"},{"line":94,"text":"    if not rows:"},{"line":95,"text":"        return empty_state"},{"line":96,"text":"    kinds = []"},{"line":97,"text":"    for row in rows:"},{"line":98,"text":"        for key in keys:"},{"line":99,"text":"            if row.get(key) is not None:"},{"line":100,"text":"                kinds.append(status_kind(row.get(key)))"},{"line":101,"text":"    if \"bad\" in kinds:"},{"line":102,"text":"        return \"ERROR\""},{"line":103,"text":"    if \"warn\" in kinds or \"unknown\" in kinds or not kinds:"},{"line":104,"text":"        return \"WARN\""},{"line":105,"text":"    return \"OK\""},{"line":106,"text":""},{"line":107,"text":"def incident_section_state(rows):"},{"line":108,"text":"    if not rows:"},{"line":109,"text":"        return \"OK\""},{"line":110,"text":"    severe = {\"P0\", \"P1\", \"CRITICAL\", \"ERROR\"}"},{"line":111,"text":"    for row in rows:"},{"line":112,"text":"        severity = str(row.get(\"severity\") or \"\").strip().upper()"},{"line":113,"text":"        state = row.get(\"status\")"},{"line":114,"text":"        if severity in severe or status_kind(state) == \"bad\":"},{"line":115,"text":"            return \"ERROR\""},{"line":116,"text":"    return \"WARN\""},{"line":117,"text":""},{"line":118,"text":"# Все текущие и будущие подробные блоки dashboard должны использовать этот компонент."},{"line":119,"text":"def health_section(title, count, state, content, wide=False):"},{"line":120,"text":"    kind = status_kind(state)"},{"line":121,"text":"    css_class = \"panel health-section\" + (\" wide\" if wide else \"\")"},{"line":122,"text":"    return \"<details class=\\\"\" + css_class + \"\\\"><summary><span class=\\\"section-heading\\\"><span class=\\\"state state-\" + kind + \"\\\" aria-label=\\\"\" + html.escape(str(state)) + \"\\\"><span class=\\\"state-dot\\\"></span></span><span class=\\\"section-title\\\">\" + html.escape(str(title)) + \"</span><span class=\\\"section-count\\\">\" + html.escape(str(count)) + \"</span></span><span class=\\\"section-chevron\\\" aria-hidden=\\\"true\\\">⌄</span></summary><div class=\\\"section-body\\\">\" + content + \"</div></details>\""},{"line":123,"text":""},{"line":124,"text":"def table(rows, columns):"},{"line":125,"text":"    if not rows:"},{"line":126,"text":"        return \"<p class=\\\"muted\\\">Данные ещё не собраны.</p>\""},{"line":127,"text":"    head = \"\".join(\"<th>\" + html.escape(str(title)) + \"</th>\" for title, key in columns)"},{"line":128,"text":"    body = []"},{"line":129,"text":"    for row in rows:"},{"line":130,"text":"        cells = []"},{"line":131,"text":"        for title, key in columns:"},{"line":132,"text":"            value = row.get(key)"},{"line":133,"text":"            rendered = status_badge(value) if key in {\"status\", \"severity\"} else html.escape(str(value if value is not None else chr(8212)))"},{"line":134,"text":"            cells.append(\"<td>\" + rendered + \"</td>\")"},{"line":135,"text":"        row_kind = status_kind(row.get(\"status\", row.get(\"severity\", \"UNKNOWN\")))"},{"line":136,"text":"        body.append(\"<tr class=\\\"row-\" + row_kind + \"\\\">\" + \"\".join(cells) + \"</tr>\")"},{"line":137,"text":"    return \"<table><thead><tr>\" + head + \"</tr></thead><tbody>\" + \"\".join(body) + \"</tbody></table>\""},{"line":138,"text":""},{"line":139,"text":"@app.middleware(\"http\")"},{"line":140,"text":"async def headers(request, call_next):"},{"line":141,"text":"    response = await call_next(request)"},{"line":142,"text":"    q = chr(39)"},{"line":143,"text":"    response.headers[\"Content-Security-Policy\"] = f\"default-src {q}self{q};style-src {q}self{q} {q}unsafe-inline{q};frame-ancestors {q}none{q};base-uri {q}none{q};form-action {q}self{q}\""},{"line":144,"text":"    response.headers[\"X-Frame-Options\"] = \"DENY\""},{"line":145,"text":"    response.headers[\"X-Content-Type-Options\"] = \"nosniff\""},{"line":146,"text":"    response.headers[\"Referrer-Policy\"] = \"no-referrer\""},{"line":147,"text":"    response.headers[\"Cache-Control\"] = \"no-store\""},{"line":148,"text":"    return response"},{"line":149,"text":""},{"line":150,"text":"@app.get(\"/health\")"},{"line":151,"text":"def health():"},{"line":152,"text":"    return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":153,"text":""},{"line":154,"text":"@app.get(\"/login\",response_class=HTMLResponse)"},{"line":155,"text":"def get_login(request:Request):"},{"line":156,"text":"    return RedirectResponse(\"/\",303) if session(request) else HTMLResponse(login_page(request))"},{"line":157,"text":""},{"line":158,"text":"@app.post(\"/login\",response_class=HTMLResponse)"},{"line":159,"text":"async def post_login(request:Request):"},{"line":160,"text":"    address, now = ip(request), time.time()"},{"line":161,"text":"    failures[address] = [stamp for stamp in failures[address] if now-stamp < 900]"},{"line":162,"text":"    if locks.get(address,0) > now:"},{"line":163,"text":"        return HTMLResponse(login_page(request,\"Слишком много попыток. Повторите позже.\"),429)"},{"line":164,"text":"    if int(request.headers.get(\"content-length\",\"0\") or 0) > 4096:"},{"line":165,"text":"        raise HTTPException(413)"},{"line":166,"text":"    form = parse_qs((await request.body()).decode(\"utf-8\",\"replace\"))"},{"line":167,"text":"    if not valid_login_token(request,form.get(\"csrf\",[\"\"])[0]) or not password_ok(form.get(\"password\",[\"\"])[0]): <REDACTED>"},{"line":168,"text":"        failures[address].append(now)"},{"line":169,"text":"        if len(failures[address]) >= 5:"},{"line":170,"text":"            locks[address], failures[address] = now+900, []"},{"line":171,"text":"        return HTMLResponse(login_page(request,\"Неверный пароль.\"),401)"},{"line":172,"text":"    failures.pop(address,None); locks.pop(address,None)"},{"line":173,"text":"    response = RedirectResponse(\"/\",303)"},{"line":174,"text":"    response.set_cookie(COOKIE,session_cookie(request),max_age=<REDACTED>"},{"line":175,"text":"    return response"},{"line":176,"text":""},{"line":177,"text":"@app.post(\"/logout\")"},{"line":178,"text":"async def logout(request:Request):"},{"line":179,"text":"    data = session(request)"},{"line":180,"text":"    if not data:"},{"line":181,"text":"        raise HTTPException(401)"},{"line":182,"text":"    form = parse_qs((await request.body()).decode(\"utf-8\",\"replace\"))"},{"line":183,"text":"    if not hmac.compare_digest(form.get(\"csrf\",[\"\"])[0],data[\"csrf\"]):"},{"line":184,"text":"        raise HTTPException(403)"},{"line":185,"text":"    response = RedirectResponse(\"/login\",303)"},{"line":186,"text":"    response.delete_cookie(COOKIE,path=<REDACTED>"},{"line":187,"text":"    return response"},{"line":188,"text":""},{"line":189,"text":"@app.get(\"/api/summary\")"},{"line":190,"text":"def api_summary(request:Request):"},{"line":191,"text":"    if not session(request):"},{"line":192,"text":"        raise HTTPException(401)"},{"line":193,"text":"    with db() as connection, connection.cursor(cursor_factory=RealDictCursor) as cursor:"},{"line":194,"text":"        cursor.execute(\"SELECT (SELECT count(*) FROM nodes) nodes,(SELECT count(*) FROM guests) guests,(SELECT count(*) FROM services) services,(SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)) incidents\",(\"RECOVERED\",\"CLOSED\"))"},{"line":195,"text":"        return dict(cursor.fetchone())"},{"line":196,"text":""},{"line":197,"text":"@app.get(\"/\",response_class=HTMLResponse)"},{"line":198,"text":"def dashboard(request:Request):"},{"line":199,"text":"    data = session(request)"},{"line":200,"text":"    if not data:"},{"line":201,"text":"        return RedirectResponse(\"/login\",303)"},{"line":202,"text":"    with db() as connection, connection.cursor(cursor_factory=RealDictCursor) as cursor:"},{"line":203,"text":"        cursor.execute(\"SELECT (SELECT count(*) FROM nodes) nodes,(SELECT count(*) FROM guests) guests,(SELECT count(*) FROM services) services,(SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)) incidents\",(\"RECOVERED\",\"CLOSED\"))"},{"line":204,"text":"        counts = dict(cursor.fetchone())"},{"line":205,"text":"        cursor.execute(\"SELECT name,role,status,last_seen FROM nodes ORDER BY name\"); nodes=[dict(x) for x in cursor.fetchall()]"},{"line":206,"text":"        cursor.execute(\"SELECT name,scope,host_name,container_name,status,last_seen FROM services ORDER BY name LIMIT 250\"); services=[dict(x) for x in cursor.fetchall()]"},{"line":207,"text":"        cursor.execute(\"SELECT severity,title,status,last_seen FROM incidents WHERE status NOT IN (%s,%s) ORDER BY last_seen DESC LIMIT 100\",(\"RECOVERED\",\"CLOSED\")); incidents=[dict(x) for x in cursor.fetchall()]"},{"line":208,"text":"    cards = \"\".join([summary_card(\"Ноды\", counts[\"nodes\"], \"OK\" if counts[\"nodes\"] > 0 else \"ERROR\"), summary_card(\"VM и CT\", counts[\"guests\"], \"OK\" if counts[\"guests\"] > 0 else \"ERROR\"), summary_card(\"Сервисы\", counts[\"services\"], \"OK\" if counts[\"services\"] > 0 else \"ERROR\"), summary_card(\"Инциденты\", counts[\"incidents\"], \"ERROR\" if counts[\"incidents\"] > 0 else \"OK\")])"},{"line":209,"text":"    csrf = data[\"csrf\"]"},{"line":210,"text":"    nodes_table = table(nodes,[(\"Нода\",\"name\"),(\"Роль\",\"role\"),(\"Статус\",\"status\"),(\"Последняя связь\",\"last_seen\")])"},{"line":211,"text":"    incidents_table = table(incidents,[(\"Severity\",\"severity\"),(\"Инцидент\",\"title\"),(\"Статус\",\"status\"),(\"Обновлён\",\"last_seen\")])"},{"line":212,"text":"    services_table = table(services,[(\"Сервис\",\"name\"),(\"Scope\",\"scope\"),(\"Хост\",\"host_name\"),(\"Контейнер\",\"container_name\"),(\"Статус\",\"status\"),(\"Последняя связь\",\"last_seen\")])"},{"line":213,"text":"    nodes_state = aggregate_state(nodes, (\"status\",), \"ERROR\")"},{"line":214,"text":"    incidents_state = incident_section_state(incidents)"},{"line":215,"text":"    services_state = aggregate_state(services, (\"status\",), \"ERROR\")"},{"line":216,"text":"    nodes_html = health_section(\"Ноды\", len(nodes), nodes_state, nodes_table)"},{"line":217,"text":"    incidents_html = health_section(\"Активные инциденты\", len(incidents), incidents_state, incidents_table)"},{"line":218,"text":"    services_html = health_section(\"Сервисы\", len(services), services_state, services_table, wide=True)"},{"line":219,"text":"    engine_badge = status_badge(\"OK\")"},{"line":220,"text":"    page = f\"\"\"<!doctype html><html lang=\"ru\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta http-equiv=\"refresh\" content=\"30\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\"post\" action=\"/logout\"><input type=\"hidden\" name=\"csrf\" value=\"{csrf}\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\"\"\""},{"line":221,"text":"    return HTMLResponse(page)"},{"line":222,"text":""},{"line":223,"text":"from fastapi.responses import PlainTextResponse"},{"line":224,"text":""},{"line":225,"text":"@app.get(\"/metrics\", response_class=PlainTextResponse)"},{"line":226,"text":"def metrics():"},{"line":227,"text":"    with db() as connection, connection.cursor() as cursor:"},{"line":228,"text":"        cursor.execute(\"SELECT count(*) FROM nodes\")"},{"line":229,"text":"        nodes = cursor.fetchone()[0]"},{"line":230,"text":"        cursor.execute(\"SELECT count(*) FROM guests\")"},{"line":231,"text":"        guests = cursor.fetchone()[0]"},{"line":232,"text":"        cursor.execute(\"SELECT count(*) FROM services\")"},{"line":233,"text":"        services = cursor.fetchone()[0]"},{"line":234,"text":"        cursor.execute(\"SELECT count(*) FROM observations\")"},{"line":235,"text":"        observations = cursor.fetchone()[0]"},{"line":236,"text":"        cursor.execute(\"SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)\", (\"RECOVERED\",\"CLOSED\"))"},{"line":237,"text":"        incidents = cursor.fetchone()[0]"},{"line":238,"text":"        cursor.execute(\"SELECT COALESCE(EXTRACT(EPOCH FROM (now()-max(finished_at))),999999) FROM collector_runs WHERE status=%s\", (\"OK\",))"},{"line":239,"text":"        collector_age = float(cursor.fetchone()[0])"},{"line":240,"text":"        cursor.execute(\"SELECT severity,count(*) FROM incidents WHERE status NOT IN (%s,%s) GROUP BY severity ORDER BY severity\", (\"RECOVERED\",\"CLOSED\"))"},{"line":241,"text":"        severity_rows = cursor.fetchall()"},{"line":242,"text":"    lines = ["},{"line":243,"text":"        \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":"        \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":"        \"cluster_admin_up 1\","},{"line":246,"text":"        \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":"        \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":"        \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":"        \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":"        \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":"        \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":"        \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":"        \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":"        \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":"        \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":"        \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":"        \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":"        \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":"        \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":"        \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":"        \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":"        \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":"        \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":264,"text":"    ]"},{"line":265,"text":"    for severity, count in severity_rows:"},{"line":266,"text":"        clean = str(severity).replace(chr(34), \"\")"},{"line":267,"text":"        lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"},{"line":268,"text":"    return PlainTextResponse(\"\\n\".join(lines) + \"\\n\", media_type=\"text/plain; version=0.0.4\")"},{"line":269,"text":""},{"line":270,"text":"@contextmanager"},{"line":271,"text":"def api_v1_read_cursor():"},{"line":272,"text":"    connection = db()"},{"line":273,"text":"    connection.autocommit = True"},{"line":274,"text":"    cursor = connection.cursor(cursor_factory=RealDictCursor)"},{"line":275,"text":"    try:"},{"line":276,"text":"        cursor.execute(\"BEGIN READ ONLY\")"},{"line":277,"text":"        cursor.execute(\"SET LOCAL statement_timeout TO 5000\")"},{"line":278,"text":"        yield cursor"},{"line":279,"text":"    finally:"},{"line":280,"text":"        try:"},{"line":281,"text":"            cursor.execute(\"ROLLBACK\")"},{"line":282,"text":"        except Exception:"},{"line":283,"text":"            pass"},{"line":284,"text":"        cursor.close()"},{"line":285,"text":"        connection.close()"},{"line":286,"text":""},{"line":287,"text":""},{"line":288,"text":"def api_v1_require_session(request: Request):"},{"line":289,"text":"    data = session(request)"},{"line":290,"text":"    if not data:"},{"line":291,"text":"        raise HTTPException(401)"},{"line":292,"text":"    return data"},{"line":293,"text":""},{"line":294,"text":""},{"line":295,"text":"def api_v1_limit(value):"},{"line":296,"text":"    if value < 1 or value > 500:"},{"line":297,"text":"        raise HTTPException(422, detail=\"limit must be between 1 and 500\")"},{"line":298,"text":"    return value"},{"line":299,"text":""},{"line":300,"text":""},{"line":301,"text":"def api_v1_updated_since(value):"},{"line":302,"text":"    if not value:"},{"line":303,"text":"        return None"},{"line":304,"text":"    try:"},{"line":305,"text":"        return datetime.fromisoformat(value.replace(\"Z\", \"+00:00\"))"},{"line":306,"text":"    except Exception:"},{"line":307,"text":"        raise HTTPException(422, detail=\"updated_since must be ISO 8601\")"},{"line":308,"text":""},{"line":309,"text":""},{"line":310,"text":"def api_v1_encode_id_cursor(value):"},{"line":311,"text":"    return enc(str(int(value)).encode())"},{"line":312,"text":""},{"line":313,"text":""},{"line":314,"text":"def api_v1_decode_id_cursor(value):"},{"line":315,"text":"    if not value:"},{"line":316,"text":"        return None"},{"line":317,"text":"    try:"},{"line":318,"text":"        result = int(dec(value).decode())"},{"line":319,"text":"        if result < 0:"},{"line":320,"text":"            raise ValueError()"},{"line":321,"text":"        return result"},{"line":322,"text":"    except Exception:"},{"line":323,"text":"        raise HTTPException(422, detail=\"invalid cursor\")"},{"line":324,"text":""},{"line":325,"text":""},{"line":326,"text":"@app.get(\"/api/v1/entities\")"},{"line":327,"text":"def api_v1_entities(request: Request, limit: int = 100, cursor: str = \"\", entity_type: str = \"\", lifecycle_status: str = \"\", source_of_truth: str = \"\", updated_since: str = \"\", search: str = \"\"):"},{"line":328,"text":"    api_v1_require_session(request)"},{"line":329,"text":"    limit = api_v1_limit(limit)"},{"line":330,"text":"    clauses = []"},{"line":331,"text":"    params = []"},{"line":332,"text":"    if cursor:"},{"line":333,"text":"        clauses.append(\"e.entity_key > %s\")"},{"line":334,"text":"        params.append(cursor)"},{"line":335,"text":"    if entity_type:"},{"line":336,"text":"        clauses.append(\"e.entity_type = %s\")"},{"line":337,"text":"        params.append(entity_type)"},{"line":338,"text":"    if lifecycle_status:"},{"line":339,"text":"        clauses.append(\"e.lifecycle_status = %s\")"},{"line":340,"text":"        params.append(lifecycle_status)"},{"line":341,"text":"    if source_of_truth:"},{"line":342,"text":"        clauses.append(\"e.source_of_truth = %s\")"},{"line":343,"text":"        params.append(source_of_truth)"},{"line":344,"text":"    since = api_v1_updated_since(updated_since)"},{"line":345,"text":"    if since is not None:"},{"line":346,"text":"        clauses.append(\"e.updated_at >= %s\")"},{"line":347,"text":"        params.append(since)"},{"line":348,"text":"    if search:"},{"line":349,"text":"        clauses.append(\"(e.entity_key ILIKE %s OR e.display_name ILIKE %s)\")"},{"line":350,"text":"        params.extend([\"%\" + search + \"%\", \"%\" + search + \"%\"])"},{"line":351,"text":"    sql = \"SELECT e.entity_key,e.entity_type,e.display_name,e.lifecycle_status,e.source_of_truth,e.created_at,e.updated_at FROM entities e\""},{"line":352,"text":"    if clauses:"},{"line":353,"text":"        sql += \" WHERE \" + \" AND \".join(clauses)"},{"line":354,"text":"    sql += \" ORDER BY e.entity_key LIMIT %s\""},{"line":355,"text":"    params.append(limit + 1)"},{"line":356,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":357,"text":"        db_cursor.execute(sql, params)"},{"line":358,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":359,"text":"    page = rows[:limit]"},{"line":360,"text":"    next_cursor = page[-1][\"entity_key\"] if len(rows) > limit else None"},{"line":361,"text":"    return {\"items\": page, \"next_cursor\": next_cursor, \"limit\": limit}"},{"line":362,"text":""},{"line":363,"text":""},{"line":364,"text":"@app.get(\"/api/v1/entities/{entity_key:path}/dependencies\")"},{"line":365,"text":"def api_v1_entity_dependencies(request: Request, entity_key: str, limit: int = 100, cursor: str = \"\"):"},{"line":366,"text":"    api_v1_require_session(request)"},{"line":367,"text":"    limit = api_v1_limit(limit)"},{"line":368,"text":"    cursor_id = api_v1_decode_id_cursor(cursor)"},{"line":369,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":370,"text":"        db_cursor.execute(\"SELECT id FROM entities WHERE entity_key=%s\", (entity_key,))"},{"line":371,"text":"        entity = db_cursor.fetchone()"},{"line":372,"text":"        if not entity:"},{"line":373,"text":"            raise HTTPException(404, detail=\"entity not found\")"},{"line":374,"text":"        entity_id = entity[\"id\"]"},{"line":375,"text":"        clauses = [\"(d.parent_entity_id=%s OR d.child_entity_id=%s)\"]"},{"line":376,"text":"        params = [entity_id, entity_id, entity_id]"},{"line":377,"text":"        if cursor_id is not None:"},{"line":378,"text":"            clauses.append(\"d.id > %s\")"},{"line":379,"text":"            params.append(cursor_id)"},{"line":380,"text":"        sql = \"SELECT d.id AS _cursor_id,CASE WHEN d.parent_entity_id=%s THEN 'outgoing' ELSE 'incoming' END AS direction,p.entity_key AS parent_entity_key,c.entity_key AS child_entity_key,d.relation_type,d.source_relation,d.criticality,d.edge_state,d.created_at,d.updated_at FROM entity_dependencies d JOIN entities p ON p.id=d.parent_entity_id JOIN entities c ON c.id=d.child_entity_id WHERE \" + \" AND \".join(clauses) + \" ORDER BY d.id LIMIT %s\""},{"line":381,"text":"        params.append(limit + 1)"},{"line":382,"text":"        db_cursor.execute(sql, params)"},{"line":383,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":384,"text":"    page = rows[:limit]"},{"line":385,"text":"    next_cursor = api_v1_encode_id_cursor(page[-1][\"_cursor_id\"]) if len(rows) > limit else None"},{"line":386,"text":"    for row in page:"},{"line":387,"text":"        row.pop(\"_cursor_id\", None)"},{"line":388,"text":"    return {\"entity_key\": entity_key, \"items\": page, \"next_cursor\": next_cursor, \"limit\": limit}"},{"line":389,"text":""},{"line":390,"text":""},{"line":391,"text":"@app.get(\"/api/v1/entities/{entity_key:path}\")"},{"line":392,"text":"def api_v1_entity_detail(request: Request, entity_key: str):"},{"line":393,"text":"    api_v1_require_session(request)"},{"line":394,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":395,"text":"        db_cursor.execute(\"SELECT id AS _entity_id,entity_key,entity_type,display_name,lifecycle_status,source_of_truth,legacy_table,created_at,updated_at FROM entities WHERE entity_key=%s\", (entity_key,))"},{"line":396,"text":"        row = db_cursor.fetchone()"},{"line":397,"text":"        if not row:"},{"line":398,"text":"            raise HTTPException(404, detail=\"entity not found\")"},{"line":399,"text":"        result = dict(row)"},{"line":400,"text":"        entity_id = result.pop(\"_entity_id\")"},{"line":401,"text":"        db_cursor.execute(\"SELECT alias,alias_type,priority,is_canonical,created_at FROM entity_aliases WHERE entity_id=%s ORDER BY is_canonical DESC,priority,alias,alias_type\", (entity_id,))"},{"line":402,"text":"        result[\"aliases\"] = [dict(alias) for alias in db_cursor.fetchall()]"},{"line":403,"text":"    return result"},{"line":404,"text":""},{"line":405,"text":""},{"line":406,"text":"@app.get(\"/api/v1/dependencies\")"},{"line":407,"text":"def api_v1_dependencies(request: Request, limit: int = 100, cursor: str = \"\", parent_entity_key: str = \"\", child_entity_key: str = \"\", relation_type: str = \"\", source_relation: str = \"\", criticality: str = \"\", edge_state: str = \"\"):"},{"line":408,"text":"    api_v1_require_session(request)"},{"line":409,"text":"    limit = api_v1_limit(limit)"},{"line":410,"text":"    cursor_id = api_v1_decode_id_cursor(cursor)"},{"line":411,"text":"    clauses = []"},{"line":412,"text":"    params = []"},{"line":413,"text":"    if cursor_id is not None:"},{"line":414,"text":"        clauses.append(\"d.id > %s\")"},{"line":415,"text":"        params.append(cursor_id)"},{"line":416,"text":"    filters = [(\"p.entity_key\", parent_entity_key), (\"c.entity_key\", child_entity_key), (\"d.relation_type\", relation_type), (\"d.source_relation\", source_relation), (\"d.criticality\", criticality), (\"d.edge_state\", edge_state)]"},{"line":417,"text":"    for column, value in filters:"},{"line":418,"text":"        if value:"},{"line":419,"text":"            clauses.append(column + \" = %s\")"},{"line":420,"text":"            params.append(value)"},{"line":421,"text":"    sql = \"SELECT d.id AS _cursor_id,p.entity_key AS parent_entity_key,c.entity_key AS child_entity_key,p.entity_type AS parent_entity_type,c.entity_type AS child_entity_type,d.relation_type,d.source_relation,d.criticality,d.edge_state,d.created_at,d.updated_at FROM entity_dependencies d JOIN entities p ON p.id=d.parent_entity_id JOIN entities c ON c.id=d.child_entity_id\""},{"line":422,"text":"    if clauses:"},{"line":423,"text":"        sql += \" WHERE \" + \" AND \".join(clauses)"},{"line":424,"text":"    sql += \" ORDER BY d.id LIMIT %s\""},{"line":425,"text":"    params.append(limit + 1)"},{"line":426,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":427,"text":"        db_cursor.execute(sql, params)"},{"line":428,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":429,"text":"    page = rows[:limit]"},{"line":430,"text":"    next_cursor = api_v1_encode_id_cursor(page[-1][\"_cursor_id\"]) if len(rows) > limit else None"},{"line":431,"text":"    for row in page:"},{"line":432,"text":"        row.pop(\"_cursor_id\", None)"},{"line":433,"text":"    return {\"items\": page, \"next_cursor\": next_cursor, \"limit\": limit}"},{"line":434,"text":""},{"line":435,"text":""},{"line":436,"text":"@app.get(\"/api/v1/sources\")"},{"line":437,"text":"def api_v1_sources(request: Request):"},{"line":438,"text":"    api_v1_require_session(request)"},{"line":439,"text":"    sql = \"SELECT s.source_key,s.source_type,s.enabled,si.instance_key,si.status AS instance_status,si.last_seen,se.expectation_type,se.interval_seconds,se.grace_seconds,se.severity,s.created_at,s.updated_at FROM sources s LEFT JOIN source_instances si ON si.source_id=s.id LEFT JOIN source_expectations se ON se.source_instance_id=si.id ORDER BY s.source_key,si.instance_key,se.expectation_type\""},{"line":440,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":441,"text":"        db_cursor.execute(sql)"},{"line":442,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":443,"text":"    return {\"items\": rows}"},{"line":444,"text":""},{"line":445,"text":""},{"line":446,"text":"@app.get(\"/api/v1/policies\")"},{"line":447,"text":"def api_v1_policies(request: Request):"},{"line":448,"text":"    api_v1_require_session(request)"},{"line":449,"text":"    sql = \"SELECT pr.policy_key,pr.scope,pr.enabled,pr.active_version,pv.document->>'mode' AS mode,pv.document->>'automatic_actions' AS automatic_actions,pv.document->>'human_approval_required' AS human_approval_required,pr.created_at,pr.updated_at FROM policy_registry pr LEFT JOIN policy_versions pv ON pv.policy_id=pr.id AND pv.version=pr.active_version ORDER BY pr.policy_key,pr.scope\""},{"line":450,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":451,"text":"        db_cursor.execute(sql)"},{"line":452,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":453,"text":"    return {\"items\": rows}"},{"line":454,"text":""},{"line":455,"text":""},{"line":456,"text":"@app.get(\"/api/v1/collector-runs\")"},{"line":457,"text":"def api_v1_collector_runs(request: Request, limit: int = 100, cursor: str = \"\", collector: str = \"\", status: str = \"\"):"},{"line":458,"text":"    api_v1_require_session(request)"},{"line":459,"text":"    limit = api_v1_limit(limit)"},{"line":460,"text":"    cursor_id = api_v1_decode_id_cursor(cursor)"},{"line":461,"text":"    clauses = []"},{"line":462,"text":"    params = []"},{"line":463,"text":"    if cursor_id is not None:"},{"line":464,"text":"        clauses.append(\"cr.id < %s\")"},{"line":465,"text":"        params.append(cursor_id)"},{"line":466,"text":"    if collector:"},{"line":467,"text":"        clauses.append(\"cr.collector = %s\")"},{"line":468,"text":"        params.append(collector)"},{"line":469,"text":"    if status:"},{"line":470,"text":"        clauses.append(\"cr.status = %s\")"},{"line":471,"text":"        params.append(status)"},{"line":472,"text":"    sql = \"SELECT cr.id AS _cursor_id,cr.collector,cr.status,cr.trigger_kind,cr.schema_version,cr.records_seen,cr.warning_count,cr.run_key,si.instance_key AS source_instance_key,cr.started_at,cr.finished_at FROM collector_runs cr LEFT JOIN source_instances si ON si.id=cr.source_instance_id\""},{"line":473,"text":"    if clauses:"},{"line":474,"text":"        sql += \" WHERE \" + \" AND \".join(clauses)"},{"line":475,"text":"    sql += \" ORDER BY cr.id DESC LIMIT %s\""},{"line":476,"text":"    params.append(limit + 1)"},{"line":477,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":478,"text":"        db_cursor.execute(sql, params)"},{"line":479,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":480,"text":"    page = rows[:limit]"},{"line":481,"text":"    next_cursor = api_v1_encode_id_cursor(page[-1][\"_cursor_id\"]) if len(rows) > limit else None"},{"line":482,"text":"    for row in page:"},{"line":483,"text":"        row.pop(\"_cursor_id\", None)"},{"line":484,"text":"    return {\"items\": page, \"next_cursor\": next_cursor, \"limit\": limit}"}]},{"bytes":10640,"line_count":113,"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","priority":50,"sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe","source":[{"line":1,"text":"#!/usr/bin/env python3"},{"line":2,"text":"import datetime"},{"line":3,"text":"import json"},{"line":4,"text":"import re"},{"line":5,"text":"import urllib.parse"},{"line":6,"text":"import urllib.request"},{"line":7,"text":"import psycopg2"},{"line":8,"text":"from psycopg2.extras import Json"},{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":11,"text":"PROMETHEUS = \"http://[PRIVATE_IP]:9090\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":13,"text":"def prometheus(expression):"},{"line":14,"text":"    try:"},{"line":15,"text":"        url = PROMETHEUS + \"/api/v1/query?\" + urllib.parse.urlencode({\"query\":expression})"},{"line":16,"text":"        with urllib.request.urlopen(url, timeout=12) as response:"},{"line":17,"text":"            body = json.loads(response.read().decode(\"utf-8\"))"},{"line":18,"text":"        return body.get(\"data\", {}).get(\"result\", []) if body.get(\"status\") == \"success\" else []"},{"line":19,"text":"    except Exception:"},{"line":20,"text":"        return []"},{"line":21,"text":"def numeric(value):"},{"line":22,"text":"    try:"},{"line":23,"text":"        return float(value)"},{"line":24,"text":"    except Exception:"},{"line":25,"text":"        return None"},{"line":26,"text":"def slug(value):"},{"line":27,"text":"    return re.sub(r\"[^a-z0-9]+\", \"-\", str(value).lower()).strip(\"-\")"},{"line":28,"text":"def entity(instance):"},{"line":29,"text":"    address = instance.rsplit(\":\", 1)[0]"},{"line":30,"text":"    return HOSTS.get(address, address)"},{"line":31,"text":"def open_incident(cursor, fingerprint, severity, title, summary, cause, metadata):"},{"line":32,"text":"    cursor.execute(\"INSERT INTO incidents(fingerprint,status,severity,title,summary,root_cause,first_seen,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,now(),now(),%s) ON CONFLICT(fingerprint) DO UPDATE SET status=EXCLUDED.status,severity=EXCLUDED.severity,title=EXCLUDED.title,summary=EXCLUDED.summary,root_cause=EXCLUDED.root_cause,last_seen=now(),recovered_at=NULL,metadata=EXCLUDED.metadata RETURNING id\",(fingerprint,\"OPEN\",severity,title,summary,cause,Json(metadata)))"},{"line":33,"text":"    incident_id = cursor.fetchone()[0]"},{"line":34,"text":"    cursor.execute(\"INSERT INTO incident_events(incident_id,event_type,message,metadata) SELECT %s,%s,%s,%s WHERE NOT EXISTS(SELECT 1 FROM incident_events WHERE incident_id=%s AND event_type=%s AND created_at>now()-make_interval(mins=>10))\",(incident_id,\"OBSERVED\",summary,Json(metadata),incident_id,\"OBSERVED\"))"},{"line":35,"text":"def main():"},{"line":36,"text":"    data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":"    connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":38,"text":"    active = set()"},{"line":39,"text":"    metric_count = 0"},{"line":40,"text":"    with connection.cursor() as cursor:"},{"line":41,"text":"        cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":42,"text":"        run_id = cursor.fetchone()[0]"},{"line":43,"text":"        for item in data.get(\"nodes\", []):"},{"line":44,"text":"            cursor.execute(\"INSERT INTO nodes(name,address,role,status,last_seen,metadata) VALUES(%s,%s,%s,%s,now(),%s) ON CONFLICT(name) DO UPDATE SET address=EXCLUDED.address,role=EXCLUDED.role,status=EXCLUDED.status,last_seen=now(),metadata=EXCLUDED.metadata\",(item.get(\"name\"),item.get(\"address\"),item.get(\"role\"),item.get(\"status\"),Json(item)))"},{"line":45,"text":"        for item in data.get(\"guests\", []):"},{"line":46,"text":"            cursor.execute(\"INSERT INTO guests(guest_key,node_name,guest_type,name,status,cpu_usage,memory_used_bytes,memory_total_bytes,disk_used_bytes,disk_total_bytes,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,now(),%s) ON CONFLICT(guest_key) DO UPDATE SET node_name=EXCLUDED.node_name,guest_type=EXCLUDED.guest_type,name=EXCLUDED.name,status=EXCLUDED.status,cpu_usage=EXCLUDED.cpu_usage,memory_used_bytes=EXCLUDED.memory_used_bytes,memory_total_bytes=EXCLUDED.memory_total_bytes,disk_used_bytes=EXCLUDED.disk_used_bytes,disk_total_bytes=EXCLUDED.disk_total_bytes,last_seen=now(),metadata=EXCLUDED.metadata\",(item.get(\"guest_key\"),item.get(\"node_name\"),item.get(\"guest_type\"),item.get(\"name\"),item.get(\"status\"),item.get(\"cpu_usage\"),item.get(\"memory_used_bytes\"),item.get(\"memory_total_bytes\"),item.get(\"disk_used_bytes\"),item.get(\"disk_total_bytes\"),Json(item)))"},{"line":47,"text":"        for item in data.get(\"services\", []):"},{"line":48,"text":"            service_key = item.get(\"service_key\") or slug(item.get(\"name\"))"},{"line":49,"text":"            cursor.execute(\"INSERT INTO services(service_key,name,scope,host_name,container_name,public_url,internal_url,status,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,now(),%s) ON CONFLICT(service_key) DO UPDATE SET name=EXCLUDED.name,scope=EXCLUDED.scope,host_name=EXCLUDED.host_name,container_name=EXCLUDED.container_name,public_url=EXCLUDED.public_url,internal_url=EXCLUDED.internal_url,status=EXCLUDED.status,last_seen=now(),metadata=EXCLUDED.metadata\",(service_key,item.get(\"name\"),item.get(\"scope\"),item.get(\"host_name\"),item.get(\"container_name\"),item.get(\"public_url\"),item.get(\"internal_url\"),item.get(\"status\"),Json(item)))"},{"line":50,"text":"            if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":51,"text":"                fingerprint = \"service:\" + service_key"},{"line":52,"text":"                active.add(fingerprint)"},{"line":53,"text":"                open_incident(cursor,fingerprint,\"P1\",\"Service requires attention: \" + str(item.get(\"name\")),\"Readiness status is \" + str(item.get(\"status\")),\"service-readiness\",item)"},{"line":54,"text":"        cursor.execute(\"DELETE FROM dependencies\")"},{"line":55,"text":"        for item in data.get(\"dependencies\", []):"},{"line":56,"text":"            cursor.execute(\"INSERT INTO dependencies(parent_key,child_key,relation,metadata) VALUES(%s,%s,%s,%s) ON CONFLICT(parent_key,child_key,relation) DO UPDATE SET metadata=EXCLUDED.metadata\",(item.get(\"parent_key\"),item.get(\"child_key\"),item.get(\"relation\"),Json(item)))"},{"line":57,"text":"        cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":58,"text":"        expressions = {\"up\":\"up\",\"cpu_used_pct\":\"100-(avg by(instance)(rate(node_cpu_seconds_total{mode=\\\"idle\\\"}[5m]))*100)\",\"memory_used_pct\":\"100*(1-(node_memory_MemAvailable_bytes/node_memory_MemTotal_bytes))\",\"filesystem_used_pct\":\"100*(1-(node_filesystem_avail_bytes{fstype!~\\\"tmpfs|overlay|squashfs|nsfs\\\"}/node_filesystem_size_bytes{fstype!~\\\"tmpfs|overlay|squashfs|nsfs\\\"}))\",\"load1\":\"node_load1\"}"},{"line":59,"text":"        for metric, expression in expressions.items():"},{"line":60,"text":"            for result in prometheus(expression):"},{"line":61,"text":"                labels = result.get(\"metric\", {})"},{"line":62,"text":"                instance = labels.get(\"instance\", \"unknown\")"},{"line":63,"text":"                entity_key = entity(instance)"},{"line":64,"text":"                value = numeric(result.get(\"value\", [None, None])[1])"},{"line":65,"text":"                if value is None:"},{"line":66,"text":"                    continue"},{"line":67,"text":"                status = \"OK\""},{"line":68,"text":"                severity = \"P2\""},{"line":69,"text":"                if metric == \"up\" and value < 1:"},{"line":70,"text":"                    status, severity = \"BAD\", \"P1\""},{"line":71,"text":"                elif metric == \"cpu_used_pct\" and value >= 97:"},{"line":72,"text":"                    status, severity = \"BAD\", \"P1\""},{"line":73,"text":"                elif metric == \"cpu_used_pct\" and value >= 90:"},{"line":74,"text":"                    status = \"WARN\""},{"line":75,"text":"                elif metric == \"memory_used_pct\" and value >= 95:"},{"line":76,"text":"                    status, severity = \"BAD\", \"P1\""},{"line":77,"text":"                elif metric == \"memory_used_pct\" and value >= 88:"},{"line":78,"text":"                    status = \"WARN\""},{"line":79,"text":"                elif metric == \"filesystem_used_pct\" and value >= 95:"},{"line":80,"text":"                    status, severity = \"BAD\", \"P1\""},{"line":81,"text":"                elif metric == \"filesystem_used_pct\" and value >= 85:"},{"line":82,"text":"                    status = \"WARN\""},{"line":83,"text":"                unit = \"percent\" if metric.endswith(\"pct\") else \"state\" if metric == \"up\" else \"load\""},{"line":84,"text":"                metadata = {\"instance\":instance,\"job\":labels.get(\"job\"),\"mountpoint\":labels.get(\"mountpoint\"),\"device\":labels.get(\"device\")}"},{"line":85,"text":"                cursor.execute(\"INSERT INTO observations(entity_type,entity_key,metric,value,unit,status,observed_at,metadata) VALUES(%s,%s,%s,%s,%s,%s,now(),%s)\",(\"target\",entity_key,metric,value,unit,status,Json(metadata)))"},{"line":86,"text":"                metric_count += 1"},{"line":87,"text":"                if status != \"OK\":"},{"line":88,"text":"                    cursor.execute(\"SELECT count(*) FROM observations WHERE entity_type=%s AND entity_key=%s AND metric=%s AND status=ANY(%s) AND observed_at>now()-make_interval(mins=>10)\",(\"target\",entity_key,metric,[\"WARN\",\"BAD\"]))"},{"line":89,"text":"                    if cursor.fetchone()[0] >= 2:"},{"line":90,"text":"                        fingerprint = \"metric:\" + slug(entity_key) + \":\" + metric + \":\" + slug(labels.get(\"mountpoint\", \"root\"))"},{"line":91,"text":"                        active.add(fingerprint)"},{"line":92,"text":"                        open_incident(cursor,fingerprint,severity,entity_key + \" - \" + metric,metric + \"=\" + format(value, \".2f\") + \" \" + unit,\"target-unavailable\" if metric == \"up\" else \"resource-pressure\",metadata)"},{"line":93,"text":"        for name, state in data.get(\"backups\", {}).items():"},{"line":94,"text":"            if state.get(\"status\") != \"success\" or not state.get(\"local_backup_deleted\", False) or not state.get(\"remote_a_job_path\") or not state.get(\"remote_b_job_path\"):"},{"line":95,"text":"                fingerprint = \"backup:\" + slug(name)"},{"line":96,"text":"                active.add(fingerprint)"},{"line":97,"text":"                open_incident(cursor,fingerprint,\"P1\",\"Backup requires attention: \" + name,\"Backup is not fully verified\",\"backup-verification\",state)"},{"line":98,"text":"        cursor.execute(\"SELECT id,fingerprint FROM incidents WHERE status=ANY(%s)\",([\"OPEN\",\"DETECTED\",\"ACKNOWLEDGED\",\"MONITORING\"],))"},{"line":99,"text":"        for incident_id, fingerprint in cursor.fetchall():"},{"line":100,"text":"            if fingerprint not in active:"},{"line":101,"text":"                cursor.execute(\"UPDATE incidents SET status=%s,recovered_at=now(),last_seen=now() WHERE id=%s\",(\"RECOVERED\",incident_id))"},{"line":102,"text":"                cursor.execute(\"INSERT INTO incident_events(incident_id,event_type,message,metadata) VALUES(%s,%s,%s,%s)\",(incident_id,\"RECOVERED\",\"Condition returned to normal\",Json({})))"},{"line":103,"text":"        cursor.execute(\"DELETE FROM observations WHERE observed_at<now()-make_interval(days=>30)\")"},{"line":104,"text":"        cursor.execute(\"UPDATE collector_runs SET status=%s,finished_at=now(),metadata=%s WHERE id=%s\",(\"OK\",Json({\"nodes\":len(data.get(\"nodes\",[])),\"guests\":len(data.get(\"guests\",[])),\"services\":len(data.get(\"services\",[])),\"metrics\":metric_count}),run_id))"},{"line":105,"text":"        cursor.execute(\"SELECT count(*) FROM incidents WHERE status=ANY(%s)\",([\"OPEN\",\"DETECTED\",\"ACKNOWLEDGED\",\"MONITORING\"],))"},{"line":106,"text":"        open_count = cursor.fetchone()[0]"},{"line":107,"text":"    connection.commit()"},{"line":108,"text":"    line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""},{"line":109,"text":"    with open(HEALTH, \"w\", encoding=\"utf-8\") as stream:"},{"line":110,"text":"        stream.write(line)"},{"line":111,"text":"    print(line.strip())"},{"line":112,"text":"if __name__ == \"__main__\":"},{"line":113,"text":"    main()"}]}],"gate":{"exact_source_count":30,"index_source_count":1,"ready_for_atomic_design":true,"reference_source_count":1,"unit_record_count":16},"hostname":"cluster-admin","index_paths":["/var/www/homelab-cluster-admin/index.html"],"input_counts":{"application_candidates":40,"health_input_files":4,"indexes":1,"reference_files":1,"units":16},"reference_paths":["/opt/homelab-cluster-admin/render-webpanel.sh"],"unit_records":[{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006178 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","Group":"clusteradmin","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:27 UTC] ; stop_time=[Wed 2026-08-05 13:20:28 UTC] ; pid=2006164 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:15 UTC] ; pid=2005776 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005777 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005778 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006179 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","Group":"","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:11:27 UTC] ; stop_time=[Wed 2026-08-05 13:11:27 UTC] ; pid=2004608 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","Group":"root","LoadState":"loaded","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","User":"root","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}]},"rollback_restored":false,"rollback_started":false,"schema":18,"status":"OK","transaction_gate":{"index_source_found":true,"next_required_action":"build corrected atomic local+remote transaction","ready_for_atomic_transaction":true,"reference_source_found":true,"remote_probe_root":true,"unit_records_found":true},"transport":{"attempts":[{"privilege":"sudo-root","rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":139487,"stdout_sha256":"a710627984d38927f5e0fdde30c8beab966ad52581868b9928ea458dc6902bf3"}],"privilege":"sudo-root","success":true},"v15_candidate_manifest":{"application_candidates":[{"bytes":133,"matches":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},{"bytes":149,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},{"bytes":154,"matches":[{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},{"bytes":151,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},{"bytes":231,"matches":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},{"bytes":154,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},{"bytes":138,"matches":[{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},{"bytes":127,"matches":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},{"bytes":616,"matches":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},{"bytes":1190,"matches":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},{"bytes":223,"matches":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},{"bytes":803,"matches":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},{"bytes":131,"matches":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},{"bytes":301,"matches":[{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},{"bytes":89,"matches":[{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f"},{"bytes":8444,"matches":[{"line":4,"text":"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')"},{"line":64,"text":" return f\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\n\""},{"line":82,"text":" old=<REDACTED>"},{"line":97,"text":"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)"},{"line":99,"text":"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))"}],"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a"},{"bytes":28599,"matches":[{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE =<REDACTED>"},{"line":14,"text":"COOKIE =<REDACTED>"},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":20,"text":"    return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":73,"text":"    return f\"\"\"<!doctype html><html lang=<REDACTED>"},{"line":152,"text":"    return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":220,"text":"    page = f\"\"\"<!doctype html><html lang=\"ru\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta http-equiv=\"refresh\" content=\"30\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\"post\" action=\"/logout\"><input type=\"hidden\" name=\"csrf\" value=\"{csrf}\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\"\"\""},{"line":243,"text":"        \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":"        \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":"        \"cluster_admin_up 1\","},{"line":246,"text":"        \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":"        \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":"        \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":"        \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":"        \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":"        \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":"        \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":"        \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":"        \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":"        \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":"        \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":"        \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":"        \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":"        \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":"        \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":"        \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":"        \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":"        \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":267,"text":"        lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6"},{"bytes":922,"matches":[{"line":2,"text":"first = getpass.getpass(\"Новый пароль Cluster Admin: \")"},{"line":12,"text":"directory = \"/etc/cluster-admin-incident-engine\""},{"line":16,"text":"account = pwd.getpwnam(\"clusteradmin\")"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3"},{"bytes":10640,"matches":[{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":36,"text":"    data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":"    connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":41,"text":"        cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":50,"text":"            if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":57,"text":"        cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":108,"text":"    line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe"},{"bytes":1857,"matches":[{"line":3,"text":"  \"type\": \"cluster-admin-stage1-foundation-migration-plan\","},{"line":43,"text":"    \"function\": \"cluster_admin_enrich_collector_run\","}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670"},{"bytes":1342,"matches":[{"line":3,"text":"  \"collector_call\": \"SELECT cluster_admin_sync_foundation(%s)\","},{"line":15,"text":"  \"function\": \"cluster_admin_sync_foundation\","},{"line":16,"text":"  \"function_signature\": \"cluster_admin_sync_foundation(bigint)\","},{"line":26,"text":"  \"type\": \"cluster-admin-stage2-migration-plan\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c"},{"bytes":2355,"matches":[{"line":51,"text":"  \"type\": \"cluster-admin-stage2-dynamic-acceptance-contract\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588"},{"bytes":2832,"matches":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},{"bytes":3471,"matches":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},{"bytes":1433,"matches":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},{"bytes":848,"matches":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":10353,"matches":[{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a"},{"bytes":168,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify"}],"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729"},{"bytes":259,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741"},{"bytes":134,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517"},{"bytes":141,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6"},{"bytes":189,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941"},{"bytes":131,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0"}],"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":199,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680"},{"bytes":272,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec"},{"bytes":380,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"}],"gate":{"application_candidate_count":40,"ready_for_remote_transaction_design":true,"reference_file_count":1,"unit_count":16},"health_input_files":[{"bytes":380,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"},{"bytes":128,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac"},{"bytes":1535,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b"}],"indexes":[{"bytes":10353,"path":"/var/www/homelab-cluster-admin/index.html","root":"/var/www/homelab-cluster-admin","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a","title_lines":["<title>Homelab Cluster Admin</title>","<h1>Homelab Cluster Admin</h1>","<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>","<div class=<REDACTED>","<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>","<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>","<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer","pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"]}],"listeners":{"rc":0,"stderr":"","stdout":"State  Recv-Q Send-Q Local Address:Port Peer Address:PortProcess                                   \nLISTEN 0      5            0.0.0.0:8080      0.0.0.0:*    users:((\"python3\",pid=439,fd=3))         \nLISTEN 0      2048         0.0.0.0:8081      0.0.0.0:*    users:((\"python3\",pid=487,fd=6))         \nLISTEN 0      244        127.0.0.1:5432      0.0.0.0:*    users:((\"postgres\",pid=469,fd=6))        \nLISTEN 0      4096   127.0.0.53%lo:53        0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=18))\nLISTEN 0      4096         0.0.0.0:5355      0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=12))\nLISTEN 0      4096      127.0.0.54:53        0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=20))\nLISTEN 0      128          0.0.0.0:22        0.0.0.0:*    users:((\"sshd\",pid=457,fd=3))            \nLISTEN 0      244            [::1]:5432         [::]:*    users:((\"postgres\",pid=469,fd=5))        \nLISTEN 0      4096            [::]:5355         [::]:*    users:((\"systemd-resolve\",pid=380,fd=14))\nLISTEN 0      128             [::]:22           [::]:*    users:((\"sshd\",pid=457,fd=4))            "},"process_lines":["    439       1 root     root     python3         /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin","    487       1 cluster+ cluster+ python3         /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1"],"reference_files":[{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"}],"units":[{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","Group":"clusteradmin","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","Group":"","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","Group":"root","LoadState":"loaded","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","User":"root","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}]},"v15_exact_result":{"changes_made":false,"control_plane_error":null,"declared_report_bytes":85056,"declared_report_sha256":"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf","identity_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":85940,"output_sha256":"c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8","output_source":"runner_json","rollback_restored":null,"rollback_started":false,"runner_document_rc":0,"runner_json_sha256":"6769d194cb102deb6fc37293d887af601bd012448bac4de49dd4bca3301f34a7","runner_rc":0,"runner_status":"OK","runner_text_sha256":"c68484da49e0164adaccfe9ecf92120c53d9324b2c3dcb96b4088376d3d177d2","runner_wrapper_matches_document":true,"sanitized_report_bytes":85006,"sanitized_report_sha256":"f05595551c37c00d473df29113cf9d597b2d3467ad5a2b50e8803e8579accc82","sanitizer_byte_delta":50,"sanitizer_changed_report":true,"state_document_sha256":"f438ecd342026e3db3f910ff87aba2e3c746147cf3fa64217fc4317774055874","state_path_sha256":"993095398c2c97878a69cd2e0aaa11e2ae1f587df8c8331835b5fd7ab7aa9de7","state_status":"OK"}}

OUTPUT_END
CHAT_OUTPUT_END
