From 16a2aa8c76d79fc307c9f5ec99e5037bfeebda08 Mon Sep 17 00:00:00 2001
From: homelab-runtime-publisher <homelab-runtime@localhost>
Date: Wed, 5 Aug 2026 13:23:09 +0000
Subject: [PATCH] runtime: publish
 HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z

---
 ...R-FACT-GAP-READ-ONLY-20260805T131000Z.json | 38 +++++++++++++++++++
 ...ER-FACT-GAP-READ-ONLY-20260805T131000Z.txt | 34 +++++++++++++++++
 runtime/latest.json                           | 16 ++++----
 runtime/latest.txt                            | 14 +++----
 4 files changed, 87 insertions(+), 15 deletions(-)
 create mode 100644 runtime/history/HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z.json
 create mode 100644 runtime/history/HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z.txt

diff --git a/runtime/history/HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z.json b/runtime/history/HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z.json
new file mode 100644
index 0000000..5041d38
--- /dev/null
+++ b/runtime/history/HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z.json
@@ -0,0 +1,38 @@
+{
+  "schema_version": 1,
+  "channel": "homelab-runtime",
+  "command_id": "HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z",
+  "status": "OK",
+  "rc": 0,
+  "host": "pve01",
+  "mode": "verify",
+  "component": "cluster-knowledge-base-error-system",
+  "started_at_utc": "2026-08-05T13:23:00Z",
+  "finished_at_utc": "2026-08-05T13:23:08Z",
+  "reference_register_checked": true,
+  "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
+  "error_register_checked": true,
+  "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
+  "command_sha256": "9088e8f5ce54b2b810894f0d1860a25390b3179b29d85f63e19e6f81c9f066a6",
+  "duplicate_failed_command_blocked": false,
+  "block_reason": null,
+  "execution_started": true,
+  "change_declared": false,
+  "result_contract_valid": true,
+  "result_contract_status": null,
+  "result_contract_error": null,
+  "command_rc": 0,
+  "changes_made": false,
+  "rollback_started": false,
+  "rollback_restored": null,
+  "mutation_outcome": "NO_MUTATION",
+  "sanitized": true,
+  "secrets_included": false,
+  "private_addresses_included": false,
+  "raw_evidence_retained_locally": true,
+  "raw_evidence_sha256": "6c2d4b05d4a1d95e2dfbc5ebcff23469e469ec2d5e1550a49d07c3ad51a5f205",
+  "sanitized_output_sha256": "6c2d4b05d4a1d95e2dfbc5ebcff23469e469ec2d5e1550a49d07c3ad51a5f205",
+  "output_truncated_in_json": false,
+  "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
+  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z\",\"command_rc\":0,\"mutation_outcome\":\"NO_MUTATION\",\"output_truncated_in_json\":false,\"rc\":0,\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\n{\"api\":{\"fetch_failure_count\":0,\"fetched_file_count\":104,\"fetched_total_bytes\":297443,\"route_coverage\":{\"blob\":true,\"branch\":true,\"repo\":true,\"tree\":true},\"text_file_count\":104,\"tree\":{\"blob_count\":125,\"body_emitted\":false,\"complete\":true,\"entry_count\":161,\"exact_commit\":\"fe1221b06643db3b00a621b0230f92a46a4edc2b\",\"json_parse_ok\":true,\"status\":200,\"tree_count\":36,\"truncated\":false}},\"authorization_sent\":true,\"authorization_values_emitted\":false,\"branch\":{\"body_emitted\":false,\"branch\":\"main\",\"commit_sha\":\"fe1221b06643db3b00a621b0230f92a46a4edc2b\",\"json_parse_ok\":true,\"matches_expected_commit\":true,\"status\":200},\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z\",\"command_rc\":0,\"credential_values_emitted\":false,\"endpoint_summary\":{\"address_literal_occurrence_count\":81,\"raw_addresses_emitted\":false,\"raw_url_occurrence_count\":6,\"raw_urls_emitted\":false,\"scope_counts\":{\"external_hostname\":2,\"known_external_gitea\":1,\"private\":1},\"tokens\":[{\"host_scope\":\"external_hostname\",\"host_sha256\":\"8005522570737cc57f67406b7c6fb5d572797e4ba947dfb0bf3befee3639e667\",\"path_depth\":2,\"port\":null,\"scheme\":\"https\"},{\"host_scope\":\"known_external_gitea\",\"host_sha256\":\"92feea413ab6bd72fdd8b9102d872777d284d002a6be3e5323ca6e848105ee88\",\"path_depth\":2,\"port\":null,\"scheme\":\"https\"},{\"host_scope\":\"external_hostname\",\"host_sha256\":\"11d6a89894efa93b767088253526b02bac35874c1f63783507a3fb1d3b194e8a\",\"path_depth\":3,\"port\":null,\"scheme\":\"https\"},{\"host_scope\":\"private\",\"host_sha256\":\"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69\",\"path_depth\":1,\"port\":8081,\"scheme\":\"http\"}],\"unique_endpoint_token_count\":4},\"error_fingerprint\":\"dadccd2e354a457c337b9089047ec0a0918a5d3cccd5a188b83f8dce425a59ab\",\"error_registry\":{\"content_sha256\":\"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64\",\"duplicate_fingerprint_count\":0,\"duplicate_id_count\":0,\"duplicate_ids\":[],\"fingerprint_value_count\":0,\"fingerprint_values_emitted\":false,\"migratable_record_count\":0,\"migration_coverage_ratio\":0.0,\"path\":\"errors/regression-cases.json\",\"present\":true,\"prevention_evidence_count\":0,\"raw_records_emitted\":false,\"record_count\":59,\"status_counts\":{},\"unique_fingerprint_count\":0,\"unique_id_count\":59},\"fact_record_summary\":{\"domain_record_counts\":{\"access\":10,\"freshness\":25,\"identity\":16},\"emitted_record_count\":25,\"raw_record_values_emitted\":false,\"secret_hashes_emitted\":false,\"source_record_counts\":{\"inventory/access-paths.json\":9,\"inventory/cluster-nodes.json\":3,\"observed/cluster-guests.json\":11,\"observed/current-context.json\":2},\"top_fields\":[\"status\",\"id\",\"node\",\"type\",\"name\",\"vmid\",\"address\",\"service\"],\"unique_record_count\":25},\"fact_records\":[],\"fact_records_omitted_but_count_retained\":true,\"files_written\":false,\"http_methods\":[\"GET\"],\"mutation_blueprint\":{\"atomicity_requirements\":[\"read all exact source blobs again at the expected branch commit\",\"build the full candidate tree in a temporary directory\",\"validate JSON, JSONL, schemas, gate self-tests and repository tests\",\"create byte-for-byte backup objects for every modified blob\",\"create one commit against the expected parent commit\",\"verify the new commit tree and CI-visible files\",\"on any failure restore the exact previous branch state\"],\"code_root_choice\":{\"basis\":\"existing_top_level_code_or_directory\",\"existing_python_file_count\":2,\"path\":\"tools/pre_command_gate.py\",\"path_exists\":false,\"root\":\"tools\"},\"files_to_create_or_replace\":[\"docs/CLUSTER-HANDBOOK.md\",\"inventory/cluster-reference.json\",\"docs/ERROR-SYSTEM-V2.md\",\"schemas/error-record.schema.json\",\"errors/error-registry.jsonl\",\"tools/pre_command_gate.py\",\"tests/test_error_system_v2.py\"],\"files_to_update_conditionally\":[\".gitea/workflows/ci.yml\",\"docs/WORKFLOW.md\"],\"migration_requirements\":{\"append_runtime_failures_from_known_error_gate\":true,\"derive_fingerprint_only_from_sanitized_stable_fields\":true,\"mark_parallel_draft_superseded_only_after_validation\":true,\"preserve_all_existing_records\":true,\"require_prevention_test_for_each_active_record\":true},\"migration_source\":\"errors/regression-cases.json\",\"ready\":true,\"transaction_preconditions\":{\"branch\":\"main\",\"exact_source_sha256\":{\".gitea/workflows/ci.yml\":\"b078ac1b57e0846a90fd6acb1f3cb0866dbed0fb3bbde0d7070d29a344907681\",\"docs/WORKFLOW.md\":\"87a6f9a0960ee7240261138ecea122fd8fbd727d87bb0b27b02c103c3bc2f122\",\"errors/regression-cases.json\":\"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64\",\"schemas/change.schema.json\":\"57fbce7c6ca1f53c5ec2f8cd604d787f01019a2184b611e3b4406b467331d0af\",\"schemas/context.schema.json\":\"84c3c29984402b25c102575b54bdb82544844e455d3d2e115fefc5f42021e043\"},\"expected_branch_commit\":\"fe1221b06643db3b00a621b0230f92a46a4edc2b\",\"proposed_paths\":[{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"docs/CLUSTER-HANDBOOK.md\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"inventory/cluster-reference.json\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"docs/ERROR-SYSTEM-V2.md\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"schemas/error-record.schema.json\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"errors/error-registry.jsonl\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"tests/test_error_system_v2.py\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"tools/pre_command_gate.py\"}],\"repository\":\"homelab-admin/homelab-ops\"}},\"mutation_outcome\":\"NO_MUTATION\",\"network_requests_made\":true,\"next_action\":\"RUN_ONE_IDEMPOTENT_GITEA_TRANSACTION_CREATE_CLUSTER_HANDBOOK_AND_ERROR_SYSTEM_V2\",\"output_truncated_in_json\":false,\"pre_command_known_error_gate\":{\"checked\":true,\"known_error_count\":28,\"known_error_ids\":[\"candidate-file-limit-600-20260805\",\"runner-wrapper-mismatch-after-success-20260805\",\"canonical-cluster-handbook-ambiguous-20260805\",\"pvesh-adapter-context-unreadable-20260805\",\"homelab-ops-bare-repository-not-found-local-20260805\",\"gitea-network-fallback-stage-not-emitted-20260805\",\"gitea-owner-hardcoded-homelab-admin-20260805\",\"gitea-git-auth-required-or-credential-unavailable-20260805\",\"gitea-process-self-match-payload-name-20260805\",\"sanitize-bool-attributeerror-20260805\",\"reader-v3-path-misclassified-as-route-20260805\",\"reader-v3-loopback-assumption-20260805\",\"reader-v3-capability-not-treated-as-secret-20260805\",\"reader-v3-systemd-unit-fed-to-python-ast-20260805\",\"reader-v3-helper-control-flow-not-traced-20260805\",\"reader-v3-post-capability-operation-not-derived-20260805\",\"reader-v3-detected-operations-not-tested-20260805\",\"reader-v3-manifest-global-not-bound-in-sandbox-20260805\",\"reader-v3-result-file-second-source-not-read-20260805\",\"reader-v3-negative-control-mixed-with-production-20260805\",\"reader-v3-manifest-schema-guessed-20260805\",\"reader-result-output-bound-after-compaction-20260805\",\"reader-v3-generation-record-shape-guessed-20260805\",\"reader-v3-result-file-assumed-standalone-json-20260805\",\"reader-v3-gitea-detail-marker-false-inventory-20260805\",\"reverse-proxy-runtime-not-visible-on-host-20260805\",\"gitea-public-owner-not-anonymously-discoverable-20260805\",\"exact-output-long-line-not-projectable-by-portal-view-20260805\"],\"previous_exact_identity_verified\":true},\"previous_exact_verification\":{\"bundle_bytes_ok\":true,\"bundle_sha256_ok\":true,\"capsule_member_match_count\":0,\"capsule_sha256_basis\":\"verified exact lane before command publication; optional bundle member count reported separately\",\"checks\":{\"changes_made_false\":true,\"command_id\":true,\"command_rc_zero\":true,\"contract_ok\":true,\"mutation_outcome_no_mutation\":true,\"projection_ok\":true,\"rollback_started_false\":true,\"runner_rc_zero\":true,\"runner_status_ok\":true},\"embedded_output_match_count\":1,\"exists\":true,\"identity_fully_verified\":true,\"member_count\":6,\"path_from_exact_result\":true,\"runner_json_match_count\":1,\"runner_text_match_count\":1,\"unsafe_member_rejected\":false},\"private_addresses_emitted\":false,\"raw_repository_contents_emitted\":false,\"rc\":0,\"read_only\":true,\"ready_for_idempotent_mutation\":true,\"regression_tests\":{\"endpoint_token_hides_host\":true,\"git_blob_oid_semantics\":true,\"known_error_gate_complete\":true,\"passed\":true,\"private_address_not_emitted\":true,\"safe_record_fingerprint_deterministic\":true,\"sanitize_bool_type_safe\":true,\"secret_value_redacted_before_record_hash\":true,\"strict_handbook_excludes_error_registry\":true},\"repository\":{\"archived\":false,\"default_branch\":\"main\",\"empty\":false,\"full_name\":\"homelab-admin/homelab-ops\",\"identity_safe\":true,\"mirror\":false,\"name\":\"homelab-ops\",\"owner\":\"homelab-admin\",\"private\":true,\"size\":3837},\"result_projection\":{\"budget_bytes\":24976,\"bytes\":24326,\"steps\":[\"source_summaries_24\",\"fact_records_32\",\"drop_json_shapes_and_limit_headings\",\"handbook_candidates_4\",\"omit_fact_records\"]},\"rollback_restored\":null,\"rollback_started\":false,\"root_cause_classification\":\"GITEA_CLUSTER_FACTS_AND_DOCUMENT_GAPS_EXACTLY_ANALYZED_MUTATION_READY\",\"schema\":1,\"secret_hashes_emitted\":false,\"source_summaries\":[{\"blob_sha\":\"7a8cb9e2dd5bc1235937aa211293b47ed4e26263\",\"bytes\":29485,\"content_sha256\":\"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64\",\"domains\":{\"access\":9,\"backup\":22,\"error_system\":38,\"freshness\":5,\"internal_external\":1,\"repair\":38,\"restore\":9,\"services\":8},\"headings\":[],\"json_parse_ok\":true,\"path\":\"errors/regression-cases.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":60},{\"blob_sha\":\"bad7037197f12ce9ce0b8be3b5723345699680b5\",\"bytes\":2289,\"content_sha256\":\"e07ccfac39f862c2bce2da0d31112e8e055ca9d3f23a01a8f33302863239203a\",\"domains\":{\"access\":1,\"backup\":0,\"error_system\":1,\"freshness\":1,\"internal_external\":2,\"repair\":0,\"restore\":0,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"changes/CR-2026-0018/regression-cases-draft.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"d072c291ee6b99466be0f4897e9d1308ee02e2b2\",\"bytes\":1134,\"content_sha256\":\"df400edc01d8201665c92e7434f8b0663eac0d34aab35e918abfb9bb873877e8\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":0,\"freshness\":2,\"internal_external\":0,\"repair\":1,\"restore\":0,\"services\":4},\"headings\":[\"Caller inventory requirements\",\"Search surfaces\",\"Required record\",\"Gate\"],\"json_parse_ok\":false,\"path\":\"changes/CR-2026-0018/caller-inventory-requirements.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"823d591344d8b8f67e213e2b890acdbd7bfc9864\",\"bytes\":211,\"content_sha256\":\"d3347712264b62c8768c952575574140c388df6d4d045c824ed9a8722bad1454\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":1,\"restore\":0,\"services\":0},\"headings\":[\"Changes\"],\"json_parse_ok\":false,\"path\":\"changes/README.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"c42de139cdf529611d50ad86291fb40df5bc0d09\",\"bytes\":1652,\"content_sha256\":\"ac5dc881dcd9fcb72644bc893e866f463c9c2b9b4544e32434e21b5d34168fb5\",\"domains\":{\"access\":9,\"backup\":0,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":0,\"restore\":0,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"inventory/access-paths.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"5a1fb00b587a813cee2a7bd98be451bd22173f5e\",\"bytes\":4952,\"content_sha256\":\"6b732f5ce3af0e8066d020bf10ee81b764b6d93ee5f08a9c8f17d0075eeced32\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":0,\"restore\":1,\"services\":22},\"headings\":[],\"json_parse_ok\":true,\"path\":\"observed/cluster-guests.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"3327f8b644b3f3e83774d5e7748a8dfa6b4f1ebf\",\"bytes\":958,\"content_sha256\":\"5e7236f3b8072ff6115ab5bc50ad448425882e86463e90d85df1a1c8eb502d65\",\"domains\":{\"access\":0,\"backup\":1,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":1,\"restore\":0,\"services\":2},\"headings\":[\"Архитектура управления\",\"Разделение ответственности\"],\"json_parse_ok\":false,\"path\":\"docs/ARCHITECTURE.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"3b034041ab628d15e74a7167b5545b366aeb4ad1\",\"bytes\":526,\"content_sha256\":\"87a6f9a0960ee7240261138ecea122fd8fbd727d87bb0b27b02c103c3bc2f122\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":0,\"freshness\":1,\"internal_external\":0,\"repair\":1,\"restore\":0,\"services\":0},\"headings\":[\"Change workflow\"],\"json_parse_ok\":false,\"path\":\"docs/WORKFLOW.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"f03826dcf6f8c6af4a8a78206c1b841ffaffd2ca\",\"bytes\":3020,\"content_sha256\":\"91c66d8ee6622d925174055599ad5ece6270c35f8d7d7967a26609f7e290c7e6\",\"domains\":{\"access\":0,\"backup\":1,\"error_system\":3,\"freshness\":4,\"internal_external\":0,\"repair\":1,\"restore\":1,\"services\":0},\"headings\":[\"Одна постоянная команда\",\"Что проверяется автоматически\",\"Правила безопасности\"],\"json_parse_ok\":false,\"path\":\"docs/skladchik-cookie-refresh.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":17},{\"blob_sha\":\"b4979003f9bb899f4a5de41ede47eacaeac48dee\",\"bytes\":2264,\"content_sha256\":\"e74454f528cdb5522e2c43d959472a04b7cc36e33da7aa5a5bd664fc79015344\",\"domains\":{\"access\":2,\"backup\":0,\"error_system\":0,\"freshness\":2,\"internal_external\":0,\"repair\":0,\"restore\":0,\"services\":1},\"headings\":[\"Skladchik Moderator Assistant\",\"Назначение\",\"Что планируется подготовить позднее\",\"Что сейчас не делать\"],\"json_parse_ok\":false,\"path\":\"docs/planned/skladchik-moderator-assistant.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"9586f62339cc657d18d033c104229458ebcadb58\",\"bytes\":1290,\"content_sha256\":\"cfb21fbf6ead4b3537b5f6f43e19602365b7b16adba5f317e48f8f19f5d40475\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":2,\"freshness\":0,\"internal_external\":1,\"repair\":1,\"restore\":0,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"policies/operating-policy.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":1},{\"blob_sha\":\"2032d3ebab4794625854b8c80d23ab72aac9998b\",\"bytes\":1436,\"content_sha256\":\"e06b7e6285eda2885ef913aa954a70abc7c1d892199a8b7cea2ada98b8edba34\",\"domains\":{\"access\":0,\"backup\":2,\"error_system\":3,\"freshness\":0,\"internal_external\":0,\"repair\":3,\"restore\":2,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"policies/cluster-cloud-quorum.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"8352f9a2024afea21049cd7700e460ee096c6c53\",\"bytes\":969,\"content_sha256\":\"57fbce7c6ca1f53c5ec2f8cd604d787f01019a2184b611e3b4406b467331d0af\",\"domains\":{\"access\":0,\"backup\":2,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":2,\"restore\":0,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"schemas/change.schema.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"7dc9476a8ca0227b4d5bf0cbd2c706c60af21119\",\"bytes\":539,\"content_sha256\":\"84c3c29984402b25c102575b54bdb82544844e455d3d2e115fefc5f42021e043\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":0,\"restore\":0,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"schemas/context.schema.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"3d6801bd6e7faefd1fc4579c41dc387f3dd1a9f0\",\"bytes\":318,\"content_sha256\":\"b078ac1b57e0846a90fd6acb1f3cb0866dbed0fb3bbde0d7070d29a344907681\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":1,\"freshness\":0,\"internal_external\":0,\"repair\":0,\"restore\":0,\"services\":1},\"headings\":[],\"json_parse_ok\":false,\"path\":\".gitea/workflows/ci.yml\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"68ae87029a1660255df02ff192e5b9206e9a6c05\",\"bytes\":13579,\"content_sha256\":\"d3108777e5094468aa3e633740ce8a56c90ca957a9a62932be896bbe784680b0\",\"domains\":{\"access\":22,\"backup\":23,\"error_system\":16,\"freshness\":3,\"internal_external\":1,\"repair\":22,\"restore\":10,\"services\":13},\"headings\":[],\"json_parse_ok\":true,\"path\":\"tasks/skladchik-reports-monitor-controlled-reauth-v1/task.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":43},{\"blob_sha\":\"6e22f5ca60d95965765c03ac1d012631aaae68b0\",\"bytes\":6750,\"content_sha256\":\"3a8fdfb6b4b64880012fcc3bc6fee4be6f101377310325541f9776de6d1623f7\",\"domains\":{\"access\":0,\"backup\":4,\"error_system\":16,\"freshness\":4,\"internal_external\":2,\"repair\":18,\"restore\":1,\"services\":28},\"headings\":[],\"json_parse_ok\":true,\"path\":\"tasks/homelab-logrotate-namespace-latch-clearance-v1/task.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"a1528e7cbc2b1027bfd7faa220bf1f096cdde1f5\",\"bytes\":3665,\"content_sha256\":\"666072447ba11d346772f62f274dadc974c4463e20f67e67f82f228f1fcf2ce8\",\"domains\":{\"access\":20,\"backup\":17,\"error_system\":1,\"freshness\":1,\"internal_external\":0,\"repair\":4,\"restore\":10,\"services\":7},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/rollback.sh\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"d2adf1bf345063da2b02040697f4358ba4ca4ae1\",\"bytes\":4667,\"content_sha256\":\"a49bcb32ac7212d7fb9768db8a277cb5525c873c84dd6fb633edae35dc6de532\",\"domains\":{\"access\":8,\"backup\":14,\"error_system\":9,\"freshness\":1,\"internal_external\":0,\"repair\":1,\"restore\":6,\"services\":10},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"c6f94aa845bc4cd8fc061e5e03e0fd7771a01bc6\",\"bytes\":7440,\"content_sha256\":\"636254d2526f0f62184d47cdb4c30c2831b11869c2000c557666f60b55d7691b\",\"domains\":{\"access\":13,\"backup\":24,\"error_system\":13,\"freshness\":0,\"internal_external\":0,\"repair\":12,\"restore\":4,\"services\":8},\"headings\":[],\"json_parse_ok\":true,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/task.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":2},{\"blob_sha\":\"9cc7d2fa3bc044d311500a5cf798273c79e39c35\",\"bytes\":4233,\"content_sha256\":\"ac0885c9762b99f360256ad3b3894f9cbcc4a8deab529a19ef264fe446bff029\",\"domains\":{\"access\":8,\"backup\":16,\"error_system\":8,\"freshness\":1,\"internal_external\":0,\"repair\":7,\"restore\":0,\"services\":11},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/verify.sh\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"774e2b0e591c9ada9aac9700a1b8427645284f1e\",\"bytes\":4665,\"content_sha256\":\"34de7dd0cc60c90c1b74e7a91334cb737d33c761882d1483d5b39f15b439c6f1\",\"domains\":{\"access\":13,\"backup\":19,\"error_system\":1,\"freshness\":1,\"internal_external\":0,\"repair\":1,\"restore\":0,\"services\":7},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":3},{\"blob_sha\":\"938db71d80350ac328ec90fe7bb5176a4d6d50ac\",\"bytes\":2867,\"content_sha256\":\"3b8687392163cf5b6f8b0874087320fe74ceaa5fee897dd0e7762d8c71476a1c\",\"domains\":{\"access\":4,\"backup\":4,\"error_system\":1,\"freshness\":0,\"internal_external\":0,\"repair\":8,\"restore\":11,\"services\":3},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tasks/skladchik-reports-monitor-controlled-reauth-v1/phases/rollback.sh\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":2},{\"blob_sha\":\"aefe7ac6860ec2107c281485b35c8cba1f1196ef\",\"bytes\":4846,\"content_sha256\":\"cfde805604d58d87d975584431aeb024bef21e863c4293d4726e5504415fe9bc\",\"domains\":{\"access\":0,\"backup\":6,\"error_system\":5,\"freshness\":2,\"internal_external\":1,\"repair\":3,\"restore\":9,\"services\":0},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":21}],\"status\":\"OK\",\"strict_handbook\":{\"candidates\":[{\"canonical_phrase_hits\":0,\"content_sha256\":\"91c66d8ee6622d925174055599ad5ece6270c35f8d7d7967a26609f7e290c7e6\",\"covered_domains\":[\"backup\",\"error_system\",\"repair\",\"restore\"],\"headings\":[\"Одна постоянная команда\",\"Что проверяется автоматически\",\"Правила безопасности\"],\"path\":\"docs/skladchik-cookie-refresh.md\",\"score\":80,\"title_signal\":false},{\"canonical_phrase_hits\":1,\"content_sha256\":\"5e7236f3b8072ff6115ab5bc50ad448425882e86463e90d85df1a1c8eb502d65\",\"covered_domains\":[\"backup\",\"repair\",\"services\"],\"headings\":[\"Архитектура управления\",\"Разделение ответственности\"],\"path\":\"docs/ARCHITECTURE.md\",\"score\":75,\"title_signal\":false},{\"canonical_phrase_hits\":1,\"content_sha256\":\"d9cb129e009176a7dde2a07f59760acddd9e17237eef563e97a55b3e62838bd1\",\"covered_domains\":[\"repair\"],\"headings\":[\"homelab-ops\",\"Порядок работы\",\"Локальная проверка\"],\"path\":\"README.md\",\"score\":55,\"title_signal\":true},{\"canonical_phrase_hits\":0,\"content_sha256\":\"e74454f528cdb5522e2c43d959472a04b7cc36e33da7aa5a5bd664fc79015344\",\"covered_domains\":[\"access\",\"services\"],\"headings\":[\"Skladchik Moderator Assistant\",\"Назначение\",\"Что планируется подготовить позднее\",\"Что сейчас не делать\",\"Условие возобновления\"],\"path\":\"docs/planned/skladchik-moderator-assistant.md\",\"score\":40,\"title_signal\":false}],\"path\":null,\"previous_loose_candidate_rejected\":\"errors/regression-cases.json\",\"qualified_count\":0,\"rejection_reason\":\"error registry is not a cluster handbook under the strict source-class definition\",\"status\":\"NOT_FOUND\",\"strict_definition\":{\"allowed_source_classes\":[\"docs\",\"inventory\",\"root index\"],\"excluded_source_classes\":[\"errors\",\"changes\",\"tasks\",\"tests\",\"workflows\",\"schemas\"],\"minimum_domains\":6,\"requires_canonical_phrase\":true,\"requires_cluster_title\":true}},\"verified_gaps\":[{\"evidence\":{\"qualified_count\":0,\"strict_status\":\"NOT_FOUND\"},\"gap_id\":\"canonical_cluster_handbook_absent\",\"remediation\":\"create one canonical docs/CLUSTER-HANDBOOK.md and link every fact to an exact machine-readable source\",\"severity\":\"BLOCKING\"},{\"evidence\":{\"path_verified_absent\":true},\"gap_id\":\"machine_readable_cluster_reference_absent\",\"remediation\":\"create inventory/cluster-reference.json with source_sha256, verified_at and stale/unknown states\",\"severity\":\"BLOCKING\"},{\"evidence\":{\"record_count\":0},\"gap_id\":\"structured_backup_restore_facts_absent\",\"remediation\":\"materialize backup_restore facts into the machine-readable cluster reference\",\"severity\":\"HIGH\"},{\"evidence\":{\"record_count\":0},\"gap_id\":\"structured_repair_facts_absent\",\"remediation\":\"materialize repair facts into the machine-readable cluster reference\",\"severity\":\"HIGH\"},{\"evidence\":{\"record_count\":0},\"gap_id\":\"structured_scope_facts_absent\",\"remediation\":\"materialize scope facts into the machine-readable cluster reference\",\"severity\":\"HIGH\"},{\"evidence\":{\"error_record_count\":59,\"fingerprint_count\":0},\"gap_id\":\"error_registry_has_no_fingerprints\",\"remediation\":\"seed stable normalized fingerprints and block duplicate failed fingerprints before publication\",\"severity\":\"BLOCKING\"},{\"evidence\":{\"error_record_count\":59,\"prevention_evidence_count\":0},\"gap_id\":\"error_registry_has_no_prevention_evidence\",\"remediation\":\"require one positive and one negative prevention test for every active error record\",\"severity\":\"BLOCKING\"},{\"evidence\":{\"ci_mentions_fingerprint\":false,\"ci_mentions_pre_command_gate\":false},\"gap_id\":\"ci_does_not_enforce_error_fingerprint_gate\",\"remediation\":\"wire the pre-command gate and registry/schema tests into .gitea/workflows/ci.yml\",\"severity\":\"BLOCKING\"},{\"evidence\":{\"both_present\":true,\"content_sha256_equal\":false},\"gap_id\":\"parallel_draft_and_canonical_error_registries_diverge\",\"remediation\":\"declare canonical ownership and mark the draft superseded after migration\",\"severity\":\"MEDIUM\"},{\"evidence\":{\"matching_path_count\":0},\"gap_id\":\"superseded_command_registry_absent\",\"remediation\":\"record superseded command_id/capsule/result relationships during post-result ingestion\",\"severity\":\"HIGH\"}],\"workflow_gate_signals\":{\"ci_mentions_error_registry\":false,\"ci_mentions_fingerprint\":false,\"ci_mentions_pre_command_gate\":false,\"ci_present\":true,\"repository_mentions_duplicate_failed_command_blocked\":false,\"repository_mentions_error_register_checked\":false,\"repository_mentions_reference_register_checked\":false}}\n"
+}
diff --git a/runtime/history/HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z.txt b/runtime/history/HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z.txt
new file mode 100644
index 0000000..687b9d6
--- /dev/null
+++ b/runtime/history/HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z.txt
@@ -0,0 +1,34 @@
+CHAT_OUTPUT_BEGIN
+COMMAND_ID=HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z
+STATUS=OK
+RC=0
+HOST=pve01
+MODE=verify
+COMPONENT=cluster-knowledge-base-error-system
+REFERENCE_REGISTER_CHECK=OK
+REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
+ERROR_REGISTER_CHECK=OK
+ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
+COMMAND_SHA256=9088e8f5ce54b2b810894f0d1860a25390b3179b29d85f63e19e6f81c9f066a6
+DUPLICATE_FAILED_COMMAND_BLOCKED=false
+EXECUTION_STARTED=true
+CHANGE_DECLARED=false
+RESULT_CONTRACT_VALID=true
+RESULT_CONTRACT_STATUS=NOT_APPLICABLE
+RESULT_CONTRACT_ERROR=NONE
+COMMAND_RC=0
+CHANGES_MADE=false
+ROLLBACK_STARTED=false
+ROLLBACK_RESTORED=null
+MUTATION_OUTCOME=NO_MUTATION
+SANITIZED=yes
+SECRETS_INCLUDED=no
+PRIVATE_ADDRESSES_INCLUDED=no
+RAW_EVIDENCE_SHA256=6c2d4b05d4a1d95e2dfbc5ebcff23469e469ec2d5e1550a49d07c3ad51a5f205
+SANITIZED_OUTPUT_SHA256=6c2d4b05d4a1d95e2dfbc5ebcff23469e469ec2d5e1550a49d07c3ad51a5f205
+OUTPUT_BEGIN
+HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z","command_rc":0,"mutation_outcome":"NO_MUTATION","output_truncated_in_json":false,"rc":0,"rollback_restored":null,"rollback_started":false,"status":"OK","version":1}
+{"api":{"fetch_failure_count":0,"fetched_file_count":104,"fetched_total_bytes":297443,"route_coverage":{"blob":true,"branch":true,"repo":true,"tree":true},"text_file_count":104,"tree":{"blob_count":125,"body_emitted":false,"complete":true,"entry_count":161,"exact_commit":"fe1221b06643db3b00a621b0230f92a46a4edc2b","json_parse_ok":true,"status":200,"tree_count":36,"truncated":false}},"authorization_sent":true,"authorization_values_emitted":false,"branch":{"body_emitted":false,"branch":"main","commit_sha":"fe1221b06643db3b00a621b0230f92a46a4edc2b","json_parse_ok":true,"matches_expected_commit":true,"status":200},"changes_made":false,"command_id":"HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z","command_rc":0,"credential_values_emitted":false,"endpoint_summary":{"address_literal_occurrence_count":81,"raw_addresses_emitted":false,"raw_url_occurrence_count":6,"raw_urls_emitted":false,"scope_counts":{"external_hostname":2,"known_external_gitea":1,"private":1},"tokens":[{"host_scope":"external_hostname","host_sha256":"8005522570737cc57f67406b7c6fb5d572797e4ba947dfb0bf3befee3639e667","path_depth":2,"port":null,"scheme":"https"},{"host_scope":"known_external_gitea","host_sha256":"92feea413ab6bd72fdd8b9102d872777d284d002a6be3e5323ca6e848105ee88","path_depth":2,"port":null,"scheme":"https"},{"host_scope":"external_hostname","host_sha256":"11d6a89894efa93b767088253526b02bac35874c1f63783507a3fb1d3b194e8a","path_depth":3,"port":null,"scheme":"https"},{"host_scope":"private","host_sha256":"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69","path_depth":1,"port":8081,"scheme":"http"}],"unique_endpoint_token_count":4},"error_fingerprint":"dadccd2e354a457c337b9089047ec0a0918a5d3cccd5a188b83f8dce425a59ab","error_registry":{"content_sha256":"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64","duplicate_fingerprint_count":0,"duplicate_id_count":0,"duplicate_ids":[],"fingerprint_value_count":0,"fingerprint_values_emitted":false,"migratable_record_count":0,"migration_coverage_ratio":0.0,"path":"errors/regression-cases.json","present":true,"prevention_evidence_count":0,"raw_records_emitted":false,"record_count":59,"status_counts":{},"unique_fingerprint_count":0,"unique_id_count":59},"fact_record_summary":{"domain_record_counts":{"access":10,"freshness":25,"identity":16},"emitted_record_count":25,"raw_record_values_emitted":false,"secret_hashes_emitted":false,"source_record_counts":{"inventory/access-paths.json":9,"inventory/cluster-nodes.json":3,"observed/cluster-guests.json":11,"observed/current-context.json":2},"top_fields":["status","id","node","type","name","vmid","address","service"],"unique_record_count":25},"fact_records":[],"fact_records_omitted_but_count_retained":true,"files_written":false,"http_methods":["GET"],"mutation_blueprint":{"atomicity_requirements":["read all exact source blobs again at the expected branch commit","build the full candidate tree in a temporary directory","validate JSON, JSONL, schemas, gate self-tests and repository tests","create byte-for-byte backup objects for every modified blob","create one commit against the expected parent commit","verify the new commit tree and CI-visible files","on any failure restore the exact previous branch state"],"code_root_choice":{"basis":"existing_top_level_code_or_directory","existing_python_file_count":2,"path":"tools/pre_command_gate.py","path_exists":false,"root":"tools"},"files_to_create_or_replace":["docs/CLUSTER-HANDBOOK.md","inventory/cluster-reference.json","docs/ERROR-SYSTEM-V2.md","schemas/error-record.schema.json","errors/error-registry.jsonl","tools/pre_command_gate.py","tests/test_error_system_v2.py"],"files_to_update_conditionally":[".gitea/workflows/ci.yml","docs/WORKFLOW.md"],"migration_requirements":{"append_runtime_failures_from_known_error_gate":true,"derive_fingerprint_only_from_sanitized_stable_fields":true,"mark_parallel_draft_superseded_only_after_validation":true,"preserve_all_existing_records":true,"require_prevention_test_for_each_active_record":true},"migration_source":"errors/regression-cases.json","ready":true,"transaction_preconditions":{"branch":"main","exact_source_sha256":{".gitea/workflows/ci.yml":"b078ac1b57e0846a90fd6acb1f3cb0866dbed0fb3bbde0d7070d29a344907681","docs/WORKFLOW.md":"87a6f9a0960ee7240261138ecea122fd8fbd727d87bb0b27b02c103c3bc2f122","errors/regression-cases.json":"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64","schemas/change.schema.json":"57fbce7c6ca1f53c5ec2f8cd604d787f01019a2184b611e3b4406b467331d0af","schemas/context.schema.json":"84c3c29984402b25c102575b54bdb82544844e455d3d2e115fefc5f42021e043"},"expected_branch_commit":"fe1221b06643db3b00a621b0230f92a46a4edc2b","proposed_paths":[{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"docs/CLUSTER-HANDBOOK.md"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"inventory/cluster-reference.json"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"docs/ERROR-SYSTEM-V2.md"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"schemas/error-record.schema.json"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"errors/error-registry.jsonl"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"tests/test_error_system_v2.py"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"tools/pre_command_gate.py"}],"repository":"homelab-admin/homelab-ops"}},"mutation_outcome":"NO_MUTATION","network_requests_made":true,"next_action":"RUN_ONE_IDEMPOTENT_GITEA_TRANSACTION_CREATE_CLUSTER_HANDBOOK_AND_ERROR_SYSTEM_V2","output_truncated_in_json":false,"pre_command_known_error_gate":{"checked":true,"known_error_count":28,"known_error_ids":["candidate-file-limit-600-20260805","runner-wrapper-mismatch-after-success-20260805","canonical-cluster-handbook-ambiguous-20260805","pvesh-adapter-context-unreadable-20260805","homelab-ops-bare-repository-not-found-local-20260805","gitea-network-fallback-stage-not-emitted-20260805","gitea-owner-hardcoded-homelab-admin-20260805","gitea-git-auth-required-or-credential-unavailable-20260805","gitea-process-self-match-payload-name-20260805","sanitize-bool-attributeerror-20260805","reader-v3-path-misclassified-as-route-20260805","reader-v3-loopback-assumption-20260805","reader-v3-capability-not-treated-as-secret-20260805","reader-v3-systemd-unit-fed-to-python-ast-20260805","reader-v3-helper-control-flow-not-traced-20260805","reader-v3-post-capability-operation-not-derived-20260805","reader-v3-detected-operations-not-tested-20260805","reader-v3-manifest-global-not-bound-in-sandbox-20260805","reader-v3-result-file-second-source-not-read-20260805","reader-v3-negative-control-mixed-with-production-20260805","reader-v3-manifest-schema-guessed-20260805","reader-result-output-bound-after-compaction-20260805","reader-v3-generation-record-shape-guessed-20260805","reader-v3-result-file-assumed-standalone-json-20260805","reader-v3-gitea-detail-marker-false-inventory-20260805","reverse-proxy-runtime-not-visible-on-host-20260805","gitea-public-owner-not-anonymously-discoverable-20260805","exact-output-long-line-not-projectable-by-portal-view-20260805"],"previous_exact_identity_verified":true},"previous_exact_verification":{"bundle_bytes_ok":true,"bundle_sha256_ok":true,"capsule_member_match_count":0,"capsule_sha256_basis":"verified exact lane before command publication; optional bundle member count reported separately","checks":{"changes_made_false":true,"command_id":true,"command_rc_zero":true,"contract_ok":true,"mutation_outcome_no_mutation":true,"projection_ok":true,"rollback_started_false":true,"runner_rc_zero":true,"runner_status_ok":true},"embedded_output_match_count":1,"exists":true,"identity_fully_verified":true,"member_count":6,"path_from_exact_result":true,"runner_json_match_count":1,"runner_text_match_count":1,"unsafe_member_rejected":false},"private_addresses_emitted":false,"raw_repository_contents_emitted":false,"rc":0,"read_only":true,"ready_for_idempotent_mutation":true,"regression_tests":{"endpoint_token_hides_host":true,"git_blob_oid_semantics":true,"known_error_gate_complete":true,"passed":true,"private_address_not_emitted":true,"safe_record_fingerprint_deterministic":true,"sanitize_bool_type_safe":true,"secret_value_redacted_before_record_hash":true,"strict_handbook_excludes_error_registry":true},"repository":{"archived":false,"default_branch":"main","empty":false,"full_name":"homelab-admin/homelab-ops","identity_safe":true,"mirror":false,"name":"homelab-ops","owner":"homelab-admin","private":true,"size":3837},"result_projection":{"budget_bytes":24976,"bytes":24326,"steps":["source_summaries_24","fact_records_32","drop_json_shapes_and_limit_headings","handbook_candidates_4","omit_fact_records"]},"rollback_restored":null,"rollback_started":false,"root_cause_classification":"GITEA_CLUSTER_FACTS_AND_DOCUMENT_GAPS_EXACTLY_ANALYZED_MUTATION_READY","schema":1,"secret_hashes_emitted":false,"source_summaries":[{"blob_sha":"7a8cb9e2dd5bc1235937aa211293b47ed4e26263","bytes":29485,"content_sha256":"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64","domains":{"access":9,"backup":22,"error_system":38,"freshness":5,"internal_external":1,"repair":38,"restore":9,"services":8},"headings":[],"json_parse_ok":true,"path":"errors/regression-cases.json","raw_content_emitted":false,"secret_key_name_occurrence_count":60},{"blob_sha":"bad7037197f12ce9ce0b8be3b5723345699680b5","bytes":2289,"content_sha256":"e07ccfac39f862c2bce2da0d31112e8e055ca9d3f23a01a8f33302863239203a","domains":{"access":1,"backup":0,"error_system":1,"freshness":1,"internal_external":2,"repair":0,"restore":0,"services":0},"headings":[],"json_parse_ok":true,"path":"changes/CR-2026-0018/regression-cases-draft.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"d072c291ee6b99466be0f4897e9d1308ee02e2b2","bytes":1134,"content_sha256":"df400edc01d8201665c92e7434f8b0663eac0d34aab35e918abfb9bb873877e8","domains":{"access":0,"backup":0,"error_system":0,"freshness":2,"internal_external":0,"repair":1,"restore":0,"services":4},"headings":["Caller inventory requirements","Search surfaces","Required record","Gate"],"json_parse_ok":false,"path":"changes/CR-2026-0018/caller-inventory-requirements.md","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"823d591344d8b8f67e213e2b890acdbd7bfc9864","bytes":211,"content_sha256":"d3347712264b62c8768c952575574140c388df6d4d045c824ed9a8722bad1454","domains":{"access":0,"backup":0,"error_system":0,"freshness":0,"internal_external":0,"repair":1,"restore":0,"services":0},"headings":["Changes"],"json_parse_ok":false,"path":"changes/README.md","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"c42de139cdf529611d50ad86291fb40df5bc0d09","bytes":1652,"content_sha256":"ac5dc881dcd9fcb72644bc893e866f463c9c2b9b4544e32434e21b5d34168fb5","domains":{"access":9,"backup":0,"error_system":0,"freshness":0,"internal_external":0,"repair":0,"restore":0,"services":0},"headings":[],"json_parse_ok":true,"path":"inventory/access-paths.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"5a1fb00b587a813cee2a7bd98be451bd22173f5e","bytes":4952,"content_sha256":"6b732f5ce3af0e8066d020bf10ee81b764b6d93ee5f08a9c8f17d0075eeced32","domains":{"access":0,"backup":0,"error_system":0,"freshness":0,"internal_external":0,"repair":0,"restore":1,"services":22},"headings":[],"json_parse_ok":true,"path":"observed/cluster-guests.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"3327f8b644b3f3e83774d5e7748a8dfa6b4f1ebf","bytes":958,"content_sha256":"5e7236f3b8072ff6115ab5bc50ad448425882e86463e90d85df1a1c8eb502d65","domains":{"access":0,"backup":1,"error_system":0,"freshness":0,"internal_external":0,"repair":1,"restore":0,"services":2},"headings":["Архитектура управления","Разделение ответственности"],"json_parse_ok":false,"path":"docs/ARCHITECTURE.md","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"3b034041ab628d15e74a7167b5545b366aeb4ad1","bytes":526,"content_sha256":"87a6f9a0960ee7240261138ecea122fd8fbd727d87bb0b27b02c103c3bc2f122","domains":{"access":0,"backup":0,"error_system":0,"freshness":1,"internal_external":0,"repair":1,"restore":0,"services":0},"headings":["Change workflow"],"json_parse_ok":false,"path":"docs/WORKFLOW.md","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"f03826dcf6f8c6af4a8a78206c1b841ffaffd2ca","bytes":3020,"content_sha256":"91c66d8ee6622d925174055599ad5ece6270c35f8d7d7967a26609f7e290c7e6","domains":{"access":0,"backup":1,"error_system":3,"freshness":4,"internal_external":0,"repair":1,"restore":1,"services":0},"headings":["Одна постоянная команда","Что проверяется автоматически","Правила безопасности"],"json_parse_ok":false,"path":"docs/skladchik-cookie-refresh.md","raw_content_emitted":false,"secret_key_name_occurrence_count":17},{"blob_sha":"b4979003f9bb899f4a5de41ede47eacaeac48dee","bytes":2264,"content_sha256":"e74454f528cdb5522e2c43d959472a04b7cc36e33da7aa5a5bd664fc79015344","domains":{"access":2,"backup":0,"error_system":0,"freshness":2,"internal_external":0,"repair":0,"restore":0,"services":1},"headings":["Skladchik Moderator Assistant","Назначение","Что планируется подготовить позднее","Что сейчас не делать"],"json_parse_ok":false,"path":"docs/planned/skladchik-moderator-assistant.md","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"9586f62339cc657d18d033c104229458ebcadb58","bytes":1290,"content_sha256":"cfb21fbf6ead4b3537b5f6f43e19602365b7b16adba5f317e48f8f19f5d40475","domains":{"access":0,"backup":0,"error_system":2,"freshness":0,"internal_external":1,"repair":1,"restore":0,"services":0},"headings":[],"json_parse_ok":true,"path":"policies/operating-policy.json","raw_content_emitted":false,"secret_key_name_occurrence_count":1},{"blob_sha":"2032d3ebab4794625854b8c80d23ab72aac9998b","bytes":1436,"content_sha256":"e06b7e6285eda2885ef913aa954a70abc7c1d892199a8b7cea2ada98b8edba34","domains":{"access":0,"backup":2,"error_system":3,"freshness":0,"internal_external":0,"repair":3,"restore":2,"services":0},"headings":[],"json_parse_ok":true,"path":"policies/cluster-cloud-quorum.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"8352f9a2024afea21049cd7700e460ee096c6c53","bytes":969,"content_sha256":"57fbce7c6ca1f53c5ec2f8cd604d787f01019a2184b611e3b4406b467331d0af","domains":{"access":0,"backup":2,"error_system":0,"freshness":0,"internal_external":0,"repair":2,"restore":0,"services":0},"headings":[],"json_parse_ok":true,"path":"schemas/change.schema.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"7dc9476a8ca0227b4d5bf0cbd2c706c60af21119","bytes":539,"content_sha256":"84c3c29984402b25c102575b54bdb82544844e455d3d2e115fefc5f42021e043","domains":{"access":0,"backup":0,"error_system":0,"freshness":0,"internal_external":0,"repair":0,"restore":0,"services":0},"headings":[],"json_parse_ok":true,"path":"schemas/context.schema.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"3d6801bd6e7faefd1fc4579c41dc387f3dd1a9f0","bytes":318,"content_sha256":"b078ac1b57e0846a90fd6acb1f3cb0866dbed0fb3bbde0d7070d29a344907681","domains":{"access":0,"backup":0,"error_system":1,"freshness":0,"internal_external":0,"repair":0,"restore":0,"services":1},"headings":[],"json_parse_ok":false,"path":".gitea/workflows/ci.yml","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"68ae87029a1660255df02ff192e5b9206e9a6c05","bytes":13579,"content_sha256":"d3108777e5094468aa3e633740ce8a56c90ca957a9a62932be896bbe784680b0","domains":{"access":22,"backup":23,"error_system":16,"freshness":3,"internal_external":1,"repair":22,"restore":10,"services":13},"headings":[],"json_parse_ok":true,"path":"tasks/skladchik-reports-monitor-controlled-reauth-v1/task.json","raw_content_emitted":false,"secret_key_name_occurrence_count":43},{"blob_sha":"6e22f5ca60d95965765c03ac1d012631aaae68b0","bytes":6750,"content_sha256":"3a8fdfb6b4b64880012fcc3bc6fee4be6f101377310325541f9776de6d1623f7","domains":{"access":0,"backup":4,"error_system":16,"freshness":4,"internal_external":2,"repair":18,"restore":1,"services":28},"headings":[],"json_parse_ok":true,"path":"tasks/homelab-logrotate-namespace-latch-clearance-v1/task.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"a1528e7cbc2b1027bfd7faa220bf1f096cdde1f5","bytes":3665,"content_sha256":"666072447ba11d346772f62f274dadc974c4463e20f67e67f82f228f1fcf2ce8","domains":{"access":20,"backup":17,"error_system":1,"freshness":1,"internal_external":0,"repair":4,"restore":10,"services":7},"headings":[],"json_parse_ok":false,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/rollback.sh","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"d2adf1bf345063da2b02040697f4358ba4ca4ae1","bytes":4667,"content_sha256":"a49bcb32ac7212d7fb9768db8a277cb5525c873c84dd6fb633edae35dc6de532","domains":{"access":8,"backup":14,"error_system":9,"freshness":1,"internal_external":0,"repair":1,"restore":6,"services":10},"headings":[],"json_parse_ok":false,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"c6f94aa845bc4cd8fc061e5e03e0fd7771a01bc6","bytes":7440,"content_sha256":"636254d2526f0f62184d47cdb4c30c2831b11869c2000c557666f60b55d7691b","domains":{"access":13,"backup":24,"error_system":13,"freshness":0,"internal_external":0,"repair":12,"restore":4,"services":8},"headings":[],"json_parse_ok":true,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/task.json","raw_content_emitted":false,"secret_key_name_occurrence_count":2},{"blob_sha":"9cc7d2fa3bc044d311500a5cf798273c79e39c35","bytes":4233,"content_sha256":"ac0885c9762b99f360256ad3b3894f9cbcc4a8deab529a19ef264fe446bff029","domains":{"access":8,"backup":16,"error_system":8,"freshness":1,"internal_external":0,"repair":7,"restore":0,"services":11},"headings":[],"json_parse_ok":false,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/verify.sh","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"774e2b0e591c9ada9aac9700a1b8427645284f1e","bytes":4665,"content_sha256":"34de7dd0cc60c90c1b74e7a91334cb737d33c761882d1483d5b39f15b439c6f1","domains":{"access":13,"backup":19,"error_system":1,"freshness":1,"internal_external":0,"repair":1,"restore":0,"services":7},"headings":[],"json_parse_ok":false,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh","raw_content_emitted":false,"secret_key_name_occurrence_count":3},{"blob_sha":"938db71d80350ac328ec90fe7bb5176a4d6d50ac","bytes":2867,"content_sha256":"3b8687392163cf5b6f8b0874087320fe74ceaa5fee897dd0e7762d8c71476a1c","domains":{"access":4,"backup":4,"error_system":1,"freshness":0,"internal_external":0,"repair":8,"restore":11,"services":3},"headings":[],"json_parse_ok":false,"path":"tasks/skladchik-reports-monitor-controlled-reauth-v1/phases/rollback.sh","raw_content_emitted":false,"secret_key_name_occurrence_count":2},{"blob_sha":"aefe7ac6860ec2107c281485b35c8cba1f1196ef","bytes":4846,"content_sha256":"cfde805604d58d87d975584431aeb024bef21e863c4293d4726e5504415fe9bc","domains":{"access":0,"backup":6,"error_system":5,"freshness":2,"internal_external":1,"repair":3,"restore":9,"services":0},"headings":[],"json_parse_ok":false,"path":"tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py","raw_content_emitted":false,"secret_key_name_occurrence_count":21}],"status":"OK","strict_handbook":{"candidates":[{"canonical_phrase_hits":0,"content_sha256":"91c66d8ee6622d925174055599ad5ece6270c35f8d7d7967a26609f7e290c7e6","covered_domains":["backup","error_system","repair","restore"],"headings":["Одна постоянная команда","Что проверяется автоматически","Правила безопасности"],"path":"docs/skladchik-cookie-refresh.md","score":80,"title_signal":false},{"canonical_phrase_hits":1,"content_sha256":"5e7236f3b8072ff6115ab5bc50ad448425882e86463e90d85df1a1c8eb502d65","covered_domains":["backup","repair","services"],"headings":["Архитектура управления","Разделение ответственности"],"path":"docs/ARCHITECTURE.md","score":75,"title_signal":false},{"canonical_phrase_hits":1,"content_sha256":"d9cb129e009176a7dde2a07f59760acddd9e17237eef563e97a55b3e62838bd1","covered_domains":["repair"],"headings":["homelab-ops","Порядок работы","Локальная проверка"],"path":"README.md","score":55,"title_signal":true},{"canonical_phrase_hits":0,"content_sha256":"e74454f528cdb5522e2c43d959472a04b7cc36e33da7aa5a5bd664fc79015344","covered_domains":["access","services"],"headings":["Skladchik Moderator Assistant","Назначение","Что планируется подготовить позднее","Что сейчас не делать","Условие возобновления"],"path":"docs/planned/skladchik-moderator-assistant.md","score":40,"title_signal":false}],"path":null,"previous_loose_candidate_rejected":"errors/regression-cases.json","qualified_count":0,"rejection_reason":"error registry is not a cluster handbook under the strict source-class definition","status":"NOT_FOUND","strict_definition":{"allowed_source_classes":["docs","inventory","root index"],"excluded_source_classes":["errors","changes","tasks","tests","workflows","schemas"],"minimum_domains":6,"requires_canonical_phrase":true,"requires_cluster_title":true}},"verified_gaps":[{"evidence":{"qualified_count":0,"strict_status":"NOT_FOUND"},"gap_id":"canonical_cluster_handbook_absent","remediation":"create one canonical docs/CLUSTER-HANDBOOK.md and link every fact to an exact machine-readable source","severity":"BLOCKING"},{"evidence":{"path_verified_absent":true},"gap_id":"machine_readable_cluster_reference_absent","remediation":"create inventory/cluster-reference.json with source_sha256, verified_at and stale/unknown states","severity":"BLOCKING"},{"evidence":{"record_count":0},"gap_id":"structured_backup_restore_facts_absent","remediation":"materialize backup_restore facts into the machine-readable cluster reference","severity":"HIGH"},{"evidence":{"record_count":0},"gap_id":"structured_repair_facts_absent","remediation":"materialize repair facts into the machine-readable cluster reference","severity":"HIGH"},{"evidence":{"record_count":0},"gap_id":"structured_scope_facts_absent","remediation":"materialize scope facts into the machine-readable cluster reference","severity":"HIGH"},{"evidence":{"error_record_count":59,"fingerprint_count":0},"gap_id":"error_registry_has_no_fingerprints","remediation":"seed stable normalized fingerprints and block duplicate failed fingerprints before publication","severity":"BLOCKING"},{"evidence":{"error_record_count":59,"prevention_evidence_count":0},"gap_id":"error_registry_has_no_prevention_evidence","remediation":"require one positive and one negative prevention test for every active error record","severity":"BLOCKING"},{"evidence":{"ci_mentions_fingerprint":false,"ci_mentions_pre_command_gate":false},"gap_id":"ci_does_not_enforce_error_fingerprint_gate","remediation":"wire the pre-command gate and registry/schema tests into .gitea/workflows/ci.yml","severity":"BLOCKING"},{"evidence":{"both_present":true,"content_sha256_equal":false},"gap_id":"parallel_draft_and_canonical_error_registries_diverge","remediation":"declare canonical ownership and mark the draft superseded after migration","severity":"MEDIUM"},{"evidence":{"matching_path_count":0},"gap_id":"superseded_command_registry_absent","remediation":"record superseded command_id/capsule/result relationships during post-result ingestion","severity":"HIGH"}],"workflow_gate_signals":{"ci_mentions_error_registry":false,"ci_mentions_fingerprint":false,"ci_mentions_pre_command_gate":false,"ci_present":true,"repository_mentions_duplicate_failed_command_blocked":false,"repository_mentions_error_register_checked":false,"repository_mentions_reference_register_checked":false}}
+
+OUTPUT_END
+CHAT_OUTPUT_END
diff --git a/runtime/latest.json b/runtime/latest.json
index 8229ff7..5041d38 100644
--- a/runtime/latest.json
+++ b/runtime/latest.json
@@ -1,19 +1,19 @@
 {
   "schema_version": 1,
   "channel": "homelab-runtime",
-  "command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z",
+  "command_id": "HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z",
   "status": "OK",
   "rc": 0,
   "host": "pve01",
   "mode": "verify",
-  "component": "homelab-cluster-admin-update-visibility",
-  "started_at_utc": "2026-08-05T13:20:59Z",
-  "finished_at_utc": "2026-08-05T13:20:59Z",
+  "component": "cluster-knowledge-base-error-system",
+  "started_at_utc": "2026-08-05T13:23:00Z",
+  "finished_at_utc": "2026-08-05T13:23:08Z",
   "reference_register_checked": true,
   "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
   "error_register_checked": true,
   "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83",
-  "command_sha256": "8e9116b7bd5174aea8762e6a52c7c3efb4eb6ea1c0e202c3bae875aaead38778",
+  "command_sha256": "9088e8f5ce54b2b810894f0d1860a25390b3179b29d85f63e19e6f81c9f066a6",
   "duplicate_failed_command_blocked": false,
   "block_reason": null,
   "execution_started": true,
@@ -30,9 +30,9 @@
   "secrets_included": false,
   "private_addresses_included": false,
   "raw_evidence_retained_locally": true,
-  "raw_evidence_sha256": "7789c8f8591d0be49d972a2812151287eb12642bd99740576b1b9d9a29f84267",
-  "sanitized_output_sha256": "5b17a087bd51a9f4e901cd820e129160e70081636d2da63d902f481305df4f99",
+  "raw_evidence_sha256": "6c2d4b05d4a1d95e2dfbc5ebcff23469e469ec2d5e1550a49d07c3ad51a5f205",
+  "sanitized_output_sha256": "6c2d4b05d4a1d95e2dfbc5ebcff23469e469ec2d5e1550a49d07c3ad51a5f205",
   "output_truncated_in_json": false,
   "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
-  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z\",\"command_rc\":0,\"control_plane_error\":null,\"exact_source_count\":30,\"index_source_count\":1,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_atomic_transaction\":true,\"reference_source_count\":1,\"remote_probe_privilege\":\"sudo-root\",\"report_bytes\":193466,\"report_sha256\":\"772f2c3f77d7909de34aa159e0e2c8c2ca5480ff51cbcd4bcb194af9019b43d0\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"unit_record_count\":16,\"v15_declared_report_sha256\":\"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf\",\"v15_exact_result_identity_confirmed\":true,\"v15_output_sha256\":\"c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8\",\"v15_sanitized_report_sha256\":\"f05595551c37c00d473df29113cf9d597b2d3467ad5a2b50e8803e8579accc82\",\"v15_sanitizer_byte_delta\":50,\"version\":1}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z\",\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"remote_exact_map\":{\"euid\":0,\"exact_manifest\":[{\"bytes\":3708,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"priority\":100,\"requested_as\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"uid\":0},{\"bytes\":10353,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"priority\":95,\"requested_as\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"uid\":0},{\"bytes\":2832,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"priority\":92,\"requested_as\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\",\"uid\":0},{\"bytes\":3471,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"priority\":92,\"requested_as\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\",\"uid\":0},{\"bytes\":1433,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"priority\":92,\"requested_as\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\",\"uid\":0},{\"bytes\":848,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"priority\":92,\"requested_as\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\",\"uid\":0},{\"error\":\"RuntimeError: unsafe file: /usr/bin/python3.11\",\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"path\":\"/usr/bin/python3.11\",\"priority\":92,\"requested_as\":\"/usr/bin/python3\"},{\"bytes\":2541,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/usr/local/sbin/homelab-stage4i-verify\",\"priority\":92,\"requested_as\":\"/usr/local/sbin/homelab-stage4i-verify\",\"sha256\":\"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d\",\"uid\":0},{\"bytes\":616,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\",\"uid\":0},{\"bytes\":231,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\",\"uid\":0},{\"bytes\":803,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\",\"uid\":0},{\"bytes\":1190,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\",\"uid\":0},{\"bytes\":223,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\",\"uid\":0},{\"bytes\":133,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\",\"uid\":0},{\"bytes\":154,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\",\"uid\":0},{\"bytes\":131,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\",\"uid\":0},{\"bytes\":149,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\",\"uid\":0},{\"bytes\":127,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\",\"uid\":0},{\"bytes\":151,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\",\"uid\":0},{\"bytes\":138,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\",\"uid\":0},{\"bytes\":154,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\",\"uid\":0},{\"bytes\":301,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"uid\":0},{\"bytes\":465,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"sha256\":\"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b\",\"uid\":0},{\"bytes\":224,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"priority\":90,\"requested_as\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"sha256\":\"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396\",\"uid\":0},{\"is_dir\":true,\"is_file\":false,\"is_symlink\":false,\"path\":\"/opt/cluster-admin-incident-engine\",\"priority\":75,\"requested_as\":\"/opt/cluster-admin-incident-engine\"},{\"bytes\":380,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"priority\":70,\"requested_as\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\",\"uid\":0},{\"bytes\":217,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"priority\":70,\"requested_as\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\",\"uid\":0},{\"bytes\":128,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"priority\":70,\"requested_as\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\",\"uid\":0},{\"bytes\":1535,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"priority\":70,\"requested_as\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\",\"uid\":0},{\"bytes\":616,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\",\"uid\":0},{\"bytes\":231,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\",\"uid\":0},{\"bytes\":803,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\",\"uid\":0},{\"bytes\":1190,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\",\"uid\":0},{\"bytes\":223,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\",\"uid\":0},{\"bytes\":133,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\",\"uid\":0},{\"bytes\":154,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\",\"uid\":0},{\"bytes\":131,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\",\"uid\":0},{\"bytes\":149,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\",\"uid\":0},{\"bytes\":127,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\",\"uid\":0},{\"bytes\":151,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\",\"uid\":0},{\"bytes\":138,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\",\"uid\":0},{\"bytes\":154,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\",\"uid\":0},{\"bytes\":301,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"uid\":0},{\"bytes\":465,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b\",\"uid\":0},{\"bytes\":224,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396\",\"uid\":0},{\"bytes\":2832,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\",\"uid\":0},{\"bytes\":3471,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\",\"uid\":0},{\"bytes\":1433,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\",\"uid\":0},{\"bytes\":3708,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"uid\":0},{\"bytes\":848,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\",\"uid\":0},{\"bytes\":10353,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"priority\":65,\"requested_as\":\"nearby\",\"sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"uid\":0},{\"bytes\":89,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"priority\":50,\"requested_as\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\",\"uid\":0},{\"bytes\":28599,\"gid\":994,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\",\"uid\":0},{\"bytes\":10640,\"gid\":994,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\",\"uid\":0},{\"bytes\":1857,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\",\"uid\":0},{\"bytes\":2355,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\",\"uid\":0},{\"bytes\":1342,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\",\"uid\":0},{\"bytes\":922,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\",\"uid\":0},{\"bytes\":8444,\"gid\":994,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"priority\":50,\"requested_as\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\",\"uid\":0},{\"bytes\":131,\"gid\":994,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"priority\":50,\"requested_as\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"5348849c615d0258d7812fdcf660611f911b97753bbab2e0fd87e5b1c2a98cf8\",\"uid\":999},{\"bytes\":259,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"275b09782748c2b8a1b762b6390a84df202e478ece7e8f377788569b2cc5b027\",\"uid\":0},{\"bytes\":168,\"gid\":994,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"5b2bc5ae640f3b7b42000bae6b5b63d76ba00a8f55f4b6cc1cc9f80871373653\",\"uid\":999},{\"bytes\":189,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\",\"uid\":0},{\"bytes\":141,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\",\"uid\":0},{\"bytes\":199,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\",\"uid\":0},{\"bytes\":272,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"49dbf42b944b78c7394d299a08400d24a92b7532fe8d9774c686fe6a59972e15\",\"uid\":0},{\"bytes\":134,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"6353b738b98d4d038684bc753014410f96cf0ace91227dad461b4a2c69ee3c29\",\"uid\":0},{\"bytes\":214,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"priority\":50,\"requested_as\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"02916ca8a08ee305709d7fa7935209a6321b0e25fb105e0f2d3e242ed66fb0f0\",\"uid\":0},{\"bytes\":214,\"gid\":0,\"is_dir\":false,\"is_file\":true,\"is_symlink\":false,\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"priority\":50,\"requested_as\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"02916ca8a08ee305709d7fa7935209a6321b0e25fb105e0f2d3e242ed66fb0f0\",\"uid\":0}],\"exact_sources\":[{\"bytes\":3708,\"line_count\":79,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"priority\":100,\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":5,\"text\":\"mkdir -p \\\"$WEB\\\"\"},{\"line\":6,\"text\":\"BAD=0\"},{\"line\":7,\"text\":\"\"},{\"line\":8,\"text\":\"line_or_missing() {\"},{\"line\":9,\"text\":\"  f=\\\"$1\\\"\"},{\"line\":10,\"text\":\"  if [ -s \\\"$f\\\" ]; then head -n1 \\\"$f\\\"; else echo \\\"STATUS=WARN TYPE=missing FILE=$f\\\"; fi\"},{\"line\":11,\"text\":\"}\"},{\"line\":12,\"text\":\"\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":20,\"text\":\"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":21,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":22,\"text\":\"\"},{\"line\":23,\"text\":\"for line in \\\"$SELF\\\" \\\"$OBSERVER\\\" \\\"$FULL\\\" \\\"$BACKUP\\\"; do\"},{\"line\":24,\"text\":\"  echo \\\"$line\\\" | grep -q '^STATUS=OK' || BAD=$((BAD+1))\"},{\"line\":25,\"text\":\"done\"},{\"line\":26,\"text\":\"echo \\\"$FULL\\\" | grep -q 'BAD=0' || BAD=$((BAD+1))\"},{\"line\":27,\"text\":\"echo \\\"$FULL\\\" | grep -q 'WARN=0' || BAD=$((BAD+1))\"},{\"line\":28,\"text\":\"echo \\\"$BACKUP\\\" | grep -q 'BAD=0' || BAD=$((BAD+1))\"},{\"line\":29,\"text\":\"echo \\\"$BACKUP\\\" | grep -q 'MAIL_CLOUD_2COPY_VERIFY_OK' || BAD=$((BAD+1))\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":31,\"text\":\"[ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":32,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":33,\"text\":\"\"},{\"line\":34,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; COLOR=\\\"#0a7f38\\\"; BADGE=\\\"OK\\\"; else STATUS=WARN; COLOR=\\\"#b26a00\\\"; BADGE=\\\"REVIEW\\\"; fi\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":37,\"text\":\"echo \\\"$STATUS_LINE\\\" > \\\"$WEB/status.txt\\\"\"},{\"line\":38,\"text\":\"\"},{\"line\":39,\"text\":\"cat > \\\"$WEB/index.html\\\" <<HTML\"},{\"line\":40,\"text\":\"<!doctype html>\"},{\"line\":41,\"text\":\"<html lang=\\\"ru\\\">\"},{\"line\":42,\"text\":\"<head>\"},{\"line\":43,\"text\":\"<meta charset=\\\"utf-8\\\">\"},{\"line\":44,\"text\":\"<meta http-equiv=\\\"refresh\\\" content=\\\"60\\\">\"},{\"line\":45,\"text\":\"<meta name=\\\"viewport\\\" content=\\\"width=device-width, initial-scale=1\\\">\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":47,\"text\":\"<style>\"},{\"line\":48,\"text\":\"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}\"},{\"line\":49,\"text\":\".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}\"},{\"line\":50,\"text\":\"h1{margin:0 0 8px}\"},{\"line\":51,\"text\":\".badge{display:inline-block;background:$COLOR;color:white;padding:8px 14px;border-radius:999px;font-weight:700}\"},{\"line\":52,\"text\":\"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}\"},{\"line\":53,\"text\":\".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}\"},{\"line\":54,\"text\":\".small{color:#aab2c0}\"},{\"line\":55,\"text\":\"</style>\"},{\"line\":56,\"text\":\"</head>\"},{\"line\":57,\"text\":\"<body>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":59,\"text\":\"<div class=\\\"small\\\">generated UTC: $TS · VM180 · [PRIVATE_IP] · refresh 60s</div>\"},{\"line\":60,\"text\":\"<p><span class=\\\"badge\\\">$BADGE</span></p>\"},{\"line\":61,\"text\":\"\"},{\"line\":62,\"text\":\"<div class=\\\"grid\\\">\"},{\"line\":63,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>$SELF\"},{\"line\":64,\"text\":\"FAILED_UNITS=$FAILED\"},{\"line\":65,\"text\":\"QGA=$QGA</pre></div>\"},{\"line\":66,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>$OBSERVER</pre></div>\"},{\"line\":67,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>$FULL</pre></div>\"},{\"line\":68,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>$BACKUP</pre></div>\"},{\"line\":69,\"text\":\"</div>\"},{\"line\":70,\"text\":\"\"},{\"line\":71,\"text\":\"<div class=\\\"card\\\"><h2>Overall</h2><pre>$OVERALL</pre></div>\"},{\"line\":72,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>$FINAL</pre></div>\"},{\"line\":73,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>$STRICT</pre></div>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":75,\"text\":\"</body>\"},{\"line\":76,\"text\":\"</html>\"},{\"line\":77,\"text\":\"HTML\"},{\"line\":78,\"text\":\"\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}]},{\"bytes\":10353,\"line_count\":80,\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"priority\":95,\"sha256\":\"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89\",\"source\":[{\"line\":1,\"text\":\"<!doctype html>\"},{\"line\":2,\"text\":\"<html lang=\\\"ru\\\">\"},{\"line\":3,\"text\":\"<head>\"},{\"line\":4,\"text\":\"<meta charset=\\\"utf-8\\\">\"},{\"line\":5,\"text\":\"<meta http-equiv=\\\"refresh\\\" content=\\\"60\\\">\"},{\"line\":6,\"text\":\"<meta name=\\\"viewport\\\" content=\\\"width=device-width, initial-scale=1\\\">\"},{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":8,\"text\":\"<style>\"},{\"line\":9,\"text\":\"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}\"},{\"line\":10,\"text\":\".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}\"},{\"line\":11,\"text\":\"h1{margin:0 0 8px}\"},{\"line\":12,\"text\":\".badge{display:inline-block;background:#b26a00;color:white;padding:8px 14px;border-radius:999px;font-weight:700}\"},{\"line\":13,\"text\":\"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}\"},{\"line\":14,\"text\":\".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}\"},{\"line\":15,\"text\":\".small{color:#aab2c0}\"},{\"line\":16,\"text\":\"</style>\"},{\"line\":17,\"text\":\"</head>\"},{\"line\":18,\"text\":\"<body>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":20,\"text\":\"<div class=\\\"small\\\">generated UTC: 2026-08-05T13:20:48Z · VM180 · [PRIVATE_IP] · refresh 60s</div>\"},{\"line\":21,\"text\":\"<p><span class=\\\"badge\\\">REVIEW</span></p>\"},{\"line\":22,\"text\":\"\"},{\"line\":23,\"text\":\"<div class=\\\"grid\\\">\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":25,\"text\":\"FAILED_UNITS=1\"},{\"line\":26,\"text\":\"QGA=active</pre></div>\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":30,\"text\":\"</div>\"},{\"line\":31,\"text\":\"\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":36,\"text\":\"generated_utc=2026-08-05T13:19:08Z\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":38,\"text\":\"\"},{\"line\":39,\"text\":\"## Restricted probes\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":44,\"text\":\"\"},{\"line\":45,\"text\":\"## Service map checks\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":47,\"text\":\"OK severity=critical kind=tcp name=pve01_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":48,\"text\":\"OK severity=critical kind=tcp name=pve01_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\"},{\"line\":49,\"text\":\"OK severity=critical kind=tcp name=pve01_health target=[PRIVATE_IP] value=9101 tcp_9101_rc=0\"},{\"line\":50,\"text\":\"OK severity=critical kind=tcp name=pve02_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":51,\"text\":\"OK severity=critical kind=tcp name=pve02_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\"},{\"line\":52,\"text\":\"OK severity=critical kind=tcp name=pve03_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":53,\"text\":\"OK severity=critical kind=tcp name=pve03_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0\"},{\"line\":54,\"text\":\"OK severity=critical kind=tcp name=dns1_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0\"},{\"line\":55,\"text\":\"OK severity=critical kind=tcp name=dns2_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0\"},{\"line\":56,\"text\":\"OK severity=critical kind=tcp name=edge_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":58,\"text\":\"OK severity=critical kind=ping name=vm130_edge_vm_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":59,\"text\":\"OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":60,\"text\":\"OK severity=critical kind=ping name=vm150_nextcloud_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":61,\"text\":\"OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":62,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\"},{\"line\":63,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":64,\"text\":\"OK severity=critical kind=ping name=vm160_forum_prod_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":65,\"text\":\"OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":66,\"text\":\"OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=0\"},{\"line\":67,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":68,\"text\":\"OK severity=critical kind=ping name=vm170_core_apps_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":69,\"text\":\"OK severity=critical kind=tcp name=vm170_core_apps_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":70,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\"},{\"line\":71,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":72,\"text\":\"OK severity=critical kind=ping name=vm171_monitoring_ping target=[PRIVATE_IP] value=- ping_rc=0\"},{\"line\":73,\"text\":\"OK severity=critical kind=tcp name=vm171_monitoring_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":74,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1\"},{\"line\":75,\"text\":\"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1\"},{\"line\":76,\"text\":\"\"},{\"line\":77,\"text\":\"## Result\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":79,\"text\":\"</body>\"},{\"line\":80,\"text\":\"</html>\"}]},{\"bytes\":2832,\"line_count\":96,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"priority\":92,\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":4,\"text\":\"exec 9>\\\"$LOCK\\\"\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":6,\"text\":\"\"},{\"line\":7,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":10,\"text\":\"REPORT=\\\"$ROOT/latest.md\\\"\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":12,\"text\":\"TMP=\\\"$REPORT.tmp\\\"\"},{\"line\":13,\"text\":\"mkdir -p \\\"$ROOT\\\" \\\"$HEALTH_DIR\\\"\"},{\"line\":14,\"text\":\"BAD=0\"},{\"line\":15,\"text\":\"WARN=0\"},{\"line\":16,\"text\":\"\"},{\"line\":17,\"text\":\"check_ping() {\"},{\"line\":18,\"text\":\"  name=\\\"$1\\\"; ip=\\\"$2\\\"\"},{\"line\":19,\"text\":\"  ping -c1 -W1 \\\"$ip\\\" >/dev/null 2>&1\"},{\"line\":20,\"text\":\"  rc=$?\"},{\"line\":21,\"text\":\"  echo \\\"$name ping rc=$rc\\\" >> \\\"$TMP\\\"\"},{\"line\":22,\"text\":\"  [ \\\"$rc\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":23,\"text\":\"}\"},{\"line\":24,\"text\":\"\"},{\"line\":25,\"text\":\"check_tcp() {\"},{\"line\":26,\"text\":\"  name=\\\"$1\\\"; ip=\\\"$2\\\"; port=\\\"$3\\\"\"},{\"line\":27,\"text\":\"  timeout 2 bash -c \\\"</dev/tcp/$ip/$port\\\" >/dev/null 2>&1\"},{\"line\":28,\"text\":\"  rc=$?\"},{\"line\":29,\"text\":\"  echo \\\"$name tcp/$port rc=$rc\\\" >> \\\"$TMP\\\"\"},{\"line\":30,\"text\":\"  [ \\\"$rc\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":31,\"text\":\"}\"},{\"line\":32,\"text\":\"\"},{\"line\":33,\"text\":\"check_http_status_ok() {\"},{\"line\":34,\"text\":\"  name=\\\"$1\\\"; url=\\\"$2\\\"\"},{\"line\":35,\"text\":\"  line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":36,\"text\":\"  rc=$?\"},{\"line\":37,\"text\":\"  echo \\\"$name http rc=$rc line=$line\\\" >> \\\"$TMP\\\"\"},{\"line\":38,\"text\":\"  echo \\\"$line\\\" | grep -q \\\"^STATUS=\\\"\"},{\"line\":39,\"text\":\"  okrc=$?\"},{\"line\":40,\"text\":\"  [ \\\"$rc\\\" = 0 ] && [ \\\"$okrc\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":41,\"text\":\"}\"},{\"line\":42,\"text\":\"\"},{\"line\":43,\"text\":\"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":44,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":46,\"text\":\"\"},{\"line\":47,\"text\":\"{\"},{\"line\":48,\"text\":\"  echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":49,\"text\":\"  echo \\\"generated_utc=$TS\\\"\"},{\"line\":50,\"text\":\"  echo\"},{\"line\":51,\"text\":\"  echo \\\"## Self\\\"\"},{\"line\":52,\"text\":\"  echo \\\"host=$(hostname 2>/dev/null)\\\"\"},{\"line\":53,\"text\":\"  echo \\\"failed_units=$FAILED\\\"\"},{\"line\":54,\"text\":\"  echo \\\"qga=$QGA\\\"\"},{\"line\":55,\"text\":\"  echo \\\"self_check=$SELF_LINE\\\"\"},{\"line\":56,\"text\":\"  echo\"},{\"line\":57,\"text\":\"  echo \\\"## Network checks\\\"\"},{\"line\":58,\"text\":\"} > \\\"$TMP\\\"\"},{\"line\":59,\"text\":\"\"},{\"line\":60,\"text\":\"[ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":61,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":62,\"text\":\"echo \\\"$SELF_LINE\\\" | grep -q \\\"^STATUS=OK\\\" || BAD=$((BAD+1))\"},{\"line\":63,\"text\":\"\"},{\"line\":64,\"text\":\"check_ping pve01 [PRIVATE_IP]\"},{\"line\":65,\"text\":\"check_tcp pve01_ssh [PRIVATE_IP] 22\"},{\"line\":66,\"text\":\"check_tcp pve01_proxmox [PRIVATE_IP] 8006\"},{\"line\":67,\"text\":\"check_tcp pve01_health [PRIVATE_IP] 9101\"},{\"line\":68,\"text\":\"\"},{\"line\":69,\"text\":\"check_ping pve02 [PRIVATE_IP]\"},{\"line\":70,\"text\":\"check_tcp pve02_ssh [PRIVATE_IP] 22\"},{\"line\":71,\"text\":\"check_tcp pve02_proxmox [PRIVATE_IP] 8006\"},{\"line\":72,\"text\":\"\"},{\"line\":73,\"text\":\"check_ping pve03 [PRIVATE_IP]\"},{\"line\":74,\"text\":\"check_tcp pve03_ssh [PRIVATE_IP] 22\"},{\"line\":75,\"text\":\"check_tcp pve03_proxmox [PRIVATE_IP] 8006\"},{\"line\":76,\"text\":\"\"},{\"line\":77,\"text\":\"check_ping edge [PRIVATE_IP]\"},{\"line\":78,\"text\":\"check_tcp edge_ssh [PRIVATE_IP] 22\"},{\"line\":79,\"text\":\"\"},{\"line\":80,\"text\":\"check_ping dns1 [PRIVATE_IP]\"},{\"line\":81,\"text\":\"check_tcp dns1_tcp53 [PRIVATE_IP] 53\"},{\"line\":82,\"text\":\"check_ping dns2 [PRIVATE_IP]\"},{\"line\":83,\"text\":\"check_tcp dns2_tcp53 [PRIVATE_IP] 53\"},{\"line\":84,\"text\":\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":86,\"text\":\"\"},{\"line\":87,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"},{\"line\":89,\"text\":\"echo \\\"$LINE\\\" | tee \\\"$HEALTH\\\" >/dev/null\"},{\"line\":90,\"text\":\"{\"},{\"line\":91,\"text\":\"  echo\"},{\"line\":92,\"text\":\"  echo \\\"## Result\\\"\"},{\"line\":93,\"text\":\"  echo \\\"$LINE\\\"\"},{\"line\":94,\"text\":\"} >> \\\"$TMP\\\"\"},{\"line\":95,\"text\":\"mv \\\"$TMP\\\" \\\"$REPORT\\\"\"},{\"line\":96,\"text\":\"cat \\\"$HEALTH\\\"\"}]},{\"bytes\":3471,\"line_count\":113,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"priority\":92,\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":7,\"text\":\"REPORT=\\\"$ROOT/full-observer-latest.md\\\"\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":10,\"text\":\"mkdir -p \\\"$ROOT\\\" \\\"$HEALTH_DIR\\\"\"},{\"line\":11,\"text\":\"exec 9>\\\"$LOCK\\\"\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":13,\"text\":\"\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":15,\"text\":\"BAD=0\"},{\"line\":16,\"text\":\"WARN=0\"},{\"line\":17,\"text\":\"INFO_OFF=0\"},{\"line\":18,\"text\":\"TOTAL=0\"},{\"line\":19,\"text\":\"CRIT_FAIL=0\"},{\"line\":20,\"text\":\"OPT_OFF=0\"},{\"line\":21,\"text\":\"PROBE_OK=0\"},{\"line\":22,\"text\":\"PROBE_BAD=0\"},{\"line\":23,\"text\":\"\"},{\"line\":24,\"text\":\"TMP=\\\"$REPORT.tmp\\\"\"},{\"line\":25,\"text\":\"{\"},{\"line\":26,\"text\":\"  echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":27,\"text\":\"  echo \\\"generated_utc=$TS\\\"\"},{\"line\":28,\"text\":\"  echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":29,\"text\":\"  echo\"},{\"line\":30,\"text\":\"  echo \\\"## Restricted probes\\\"\"},{\"line\":31,\"text\":\"} > \\\"$TMP\\\"\"},{\"line\":32,\"text\":\"\"},{\"line\":33,\"text\":\"for t in \\\"root@[PRIVATE_IP] pve01\\\" \\\"root@[PRIVATE_IP] pve02\\\" \\\"root@[PRIVATE_IP] pve03\\\" \\\"debian@[PRIVATE_IP] edge\\\"; do\"},{\"line\":34,\"text\":\"  set -- $t\"},{\"line\":35,\"text\":\"  target=\\\"$1\\\"; name=\\\"$2\\\"\"},{\"line\":36,\"text\":\"  out=$(ssh -n -i \\\"$K\\\" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 \\\"$target\\\" probe 2>/tmp/full-probe-$name.err)\"},{\"line\":37,\"text\":\"  rc=$?\"},{\"line\":38,\"text\":\"  echo \\\"$name ssh_rc=$rc $out\\\" >> \\\"$TMP\\\"\"},{\"line\":39,\"text\":\"  echo \\\"$out\\\" | grep -q '^STATUS=OK'\"},{\"line\":40,\"text\":\"  ok=$?\"},{\"line\":41,\"text\":\"  if [ \\\"$rc\\\" = 0 ] && [ \\\"$ok\\\" = 0 ]; then PROBE_OK=$((PROBE_OK+1)); else PROBE_BAD=$((PROBE_BAD+1)); BAD=$((BAD+1)); fi\"},{\"line\":42,\"text\":\"done\"},{\"line\":43,\"text\":\"\"},{\"line\":44,\"text\":\"{\"},{\"line\":45,\"text\":\"  echo\"},{\"line\":46,\"text\":\"  echo \\\"## Service map checks\\\"\"},{\"line\":47,\"text\":\"} >> \\\"$TMP\\\"\"},{\"line\":48,\"text\":\"\"},{\"line\":49,\"text\":\"while IFS=\\\"$(printf '\\\\t')\\\" read -r severity kind name target value; do\"},{\"line\":50,\"text\":\"  [ -n \\\"$severity\\\" ] || continue\"},{\"line\":51,\"text\":\"  TOTAL=$((TOTAL+1))\"},{\"line\":52,\"text\":\"  rc=0\"},{\"line\":53,\"text\":\"  detail=\\\"\\\"\"},{\"line\":54,\"text\":\"  case \\\"$kind\\\" in\"},{\"line\":55,\"text\":\"    ping)\"},{\"line\":56,\"text\":\"      ping -c1 -W1 \\\"$target\\\" >/dev/null 2>&1\"},{\"line\":57,\"text\":\"      rc=$?\"},{\"line\":58,\"text\":\"      detail=\\\"ping_rc=$rc\\\"\"},{\"line\":59,\"text\":\"      ;;\"},{\"line\":60,\"text\":\"    tcp)\"},{\"line\":61,\"text\":\"      timeout 2 bash -c \\\"</dev/tcp/$target/$value\\\" >/dev/null 2>&1\"},{\"line\":62,\"text\":\"      rc=$?\"},{\"line\":63,\"text\":\"      detail=\\\"tcp_${value}_rc=$rc\\\"\"},{\"line\":64,\"text\":\"      ;;\"},{\"line\":65,\"text\":\"    http_status_ok)\"},{\"line\":66,\"text\":\"      line=$(curl -fsS --max-time 5 \\\"$value\\\" 2>/tmp/full-http-$name.err | head -n1)\"},{\"line\":67,\"text\":\"      curlrc=$?\"},{\"line\":68,\"text\":\"      echo \\\"$line\\\" | grep -q '^STATUS='\"},{\"line\":69,\"text\":\"      okrc=$?\"},{\"line\":70,\"text\":\"      [ \\\"$curlrc\\\" = 0 ] && [ \\\"$okrc\\\" = 0 ]\"},{\"line\":71,\"text\":\"      rc=$?\"},{\"line\":72,\"text\":\"      detail=\\\"http_rc=$curlrc ok_rc=$okrc line=$line\\\"\"},{\"line\":73,\"text\":\"      ;;\"},{\"line\":74,\"text\":\"    missing)\"},{\"line\":75,\"text\":\"      rc=1\"},{\"line\":76,\"text\":\"      detail=\\\"missing_target=$target value=$value\\\"\"},{\"line\":77,\"text\":\"      ;;\"},{\"line\":78,\"text\":\"    *)\"},{\"line\":79,\"text\":\"      rc=1\"},{\"line\":80,\"text\":\"      detail=\\\"unknown_kind=$kind\\\"\"},{\"line\":81,\"text\":\"      ;;\"},{\"line\":82,\"text\":\"  esac\"},{\"line\":83,\"text\":\"\"},{\"line\":84,\"text\":\"  if [ \\\"$rc\\\" = 0 ]; then\"},{\"line\":85,\"text\":\"    echo \\\"OK severity=$severity kind=$kind name=$name target=$target value=$value $detail\\\" >> \\\"$TMP\\\"\"},{\"line\":86,\"text\":\"  else\"},{\"line\":87,\"text\":\"    if [ \\\"$severity\\\" = critical ]; then\"},{\"line\":88,\"text\":\"      echo \\\"FAIL severity=critical kind=$kind name=$name target=$target value=$value $detail\\\" >> \\\"$TMP\\\"\"},{\"line\":89,\"text\":\"      CRIT_FAIL=$((CRIT_FAIL+1))\"},{\"line\":90,\"text\":\"      BAD=$((BAD+1))\"},{\"line\":91,\"text\":\"    else\"},{\"line\":92,\"text\":\"      echo \\\"INFO_OFF severity=optional kind=$kind name=$name target=$target value=$value $detail\\\" >> \\\"$TMP\\\"\"},{\"line\":93,\"text\":\"      OPT_OFF=$((OPT_OFF+1))\"},{\"line\":94,\"text\":\"      INFO_OFF=$((INFO_OFF+1))\"},{\"line\":95,\"text\":\"    fi\"},{\"line\":96,\"text\":\"  fi\"},{\"line\":97,\"text\":\"done < \\\"$MAP\\\"\"},{\"line\":98,\"text\":\"\"},{\"line\":99,\"text\":\"SELF_FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":100,\"text\":\"[ \\\"$SELF_FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":101,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":102,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":103,\"text\":\"\"},{\"line\":104,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"},{\"line\":106,\"text\":\"echo \\\"$LINE\\\" | tee \\\"$HEALTH\\\" >/dev/null\"},{\"line\":107,\"text\":\"{\"},{\"line\":108,\"text\":\"  echo\"},{\"line\":109,\"text\":\"  echo \\\"## Result\\\"\"},{\"line\":110,\"text\":\"  echo \\\"$LINE\\\"\"},{\"line\":111,\"text\":\"} >> \\\"$TMP\\\"\"},{\"line\":112,\"text\":\"mv \\\"$TMP\\\" \\\"$REPORT\\\"\"},{\"line\":113,\"text\":\"cat \\\"$HEALTH\\\"\"}]},{\"bytes\":1433,\"line_count\":27,\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"priority\":92,\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set -Eeuo pipefail\"},{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"},{\"line\":6,\"text\":\"ORIGINAL_RC=0\"},{\"line\":7,\"text\":\"\\\"$ORIGINAL\\\" || ORIGINAL_RC=$?\"},{\"line\":8,\"text\":\"[ -s \\\"$HEALTH\\\" ] || exit \\\"$ORIGINAL_RC\\\"\"},{\"line\":9,\"text\":\"line=\\\"$(sed -n \\\"1p\\\" \\\"$HEALTH\\\" 2>/dev/null || true)\\\"\"},{\"line\":10,\"text\":\"field(){ local key=<REDACTED>\"},{\"line\":11,\"text\":\"self_failed=\\\"$(field SELF_FAILED_UNITS || true)\\\"\"},{\"line\":12,\"text\":\"qga_state=\\\"$(field QGA || true)\\\"\"},{\"line\":13,\"text\":\"backup_state=\\\"$(field BACKUP || true)\\\"\"},{\"line\":14,\"text\":\"if [ \\\"$ORIGINAL_RC\\\" -eq 0 ] && [ \\\"$self_failed\\\" = \\\"0\\\" ] && [ \\\"$qga_state\\\" = \\\"active\\\" ] && [ \\\"$backup_state\\\" = \\\"MAIL_CLOUD_2COPY_VERIFY_OK\\\" ]; then\"},{\"line\":15,\"text\":\"  normalized=\\\"$(printf \\\"%s\\\\n\\\" \\\"$line\\\" | sed -E \\\"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\\\")\\\"\"},{\"line\":16,\"text\":\"  printf \\\"%s\\\\n\\\" \\\"$normalized\\\" > \\\"$HEALTH\\\"\"},{\"line\":17,\"text\":\"  if [ -f \\\"$STATUS_FILE\\\" ]; then\"},{\"line\":18,\"text\":\"    status_line=\\\"$(sed -n \\\"1p\\\" \\\"$STATUS_FILE\\\" 2>/dev/null || true)\\\"\"},{\"line\":19,\"text\":\"    status_normalized=\\\"$(printf \\\"%s\\\\n\\\" \\\"$status_line\\\" | sed -E \\\"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\\\")\\\"\"},{\"line\":20,\"text\":\"    tmp=\\\"${STATUS_FILE}.tmp.$$\\\"\"},{\"line\":21,\"text\":\"    { printf \\\"%s\\\\n\\\" \\\"$status_normalized\\\"; tail -n +2 \\\"$STATUS_FILE\\\" 2>/dev/null || true; } > \\\"$tmp\\\"\"},{\"line\":22,\"text\":\"    chown --reference=\\\"$STATUS_FILE\\\" \\\"$tmp\\\" 2>/dev/null || true\"},{\"line\":23,\"text\":\"    chmod --reference=\\\"$STATUS_FILE\\\" \\\"$tmp\\\" 2>/dev/null || true\"},{\"line\":24,\"text\":\"    mv \\\"$tmp\\\" \\\"$STATUS_FILE\\\"\"},{\"line\":25,\"text\":\"  fi\"},{\"line\":26,\"text\":\"fi\"},{\"line\":27,\"text\":\"exit \\\"$ORIGINAL_RC\\\"\"}]},{\"bytes\":848,\"line_count\":17,\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"priority\":92,\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set +e\"},{\"line\":3,\"text\":\"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"},{\"line\":4,\"text\":\"BAD=0\"},{\"line\":5,\"text\":\"HOST=$(hostname 2>/dev/null || echo unknown)\"},{\"line\":6,\"text\":\"IP_OK=0\"},{\"line\":7,\"text\":\"ip -4 -br a | grep -q \\\"[PRIVATE_IP]\\\" && IP_OK=1 || BAD=$((BAD+1))\"},{\"line\":8,\"text\":\"DNS_OK=0\"},{\"line\":9,\"text\":\"getent hosts pve01 >/dev/null 2>&1 || getent hosts gram1.ru >/dev/null 2>&1\"},{\"line\":10,\"text\":\"[ \\\"$?\\\" = 0 ] && DNS_OK=1 || BAD=$((BAD+1))\"},{\"line\":11,\"text\":\"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)\"},{\"line\":12,\"text\":\"[ \\\"$FAILED\\\" = 0 ] || BAD=$((BAD+1))\"},{\"line\":13,\"text\":\"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)\"},{\"line\":14,\"text\":\"[ \\\"$QGA\\\" = active ] || BAD=$((BAD+1))\"},{\"line\":15,\"text\":\"if [ \\\"$BAD\\\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi\"},{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}]},{\"bytes\":2541,\"line_count\":105,\"mode\":\"0o750\",\"path\":\"/usr/local/sbin/homelab-stage4i-verify\",\"priority\":92,\"sha256\":\"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env bash\"},{\"line\":2,\"text\":\"set -euo pipefail\"},{\"line\":3,\"text\":\"export LC_ALL=C\"},{\"line\":4,\"text\":\"\"},{\"line\":5,\"text\":\"MARKER=\\\"CR0019_STAGE4I_IMMUTABLE_V1\\\"\"},{\"line\":6,\"text\":\"STATE_DIR=\\\"/var/lib/homelab-cluster-admin/stage4i-verification\\\"\"},{\"line\":7,\"text\":\"REPORT=\\\"$STATE_DIR/latest.env\\\"\"},{\"line\":8,\"text\":\"TMP=\\\"$(mktemp)\\\"\"},{\"line\":9,\"text\":\"\"},{\"line\":10,\"text\":\"cleanup() {\"},{\"line\":11,\"text\":\"    rm -f \\\"$TMP\\\"\"},{\"line\":12,\"text\":\"}\"},{\"line\":13,\"text\":\"trap cleanup EXIT\"},{\"line\":14,\"text\":\"\"},{\"line\":15,\"text\":\"mapfile -t ACTIVATION_FILES < <(\"},{\"line\":16,\"text\":\"    find \\\\\"},{\"line\":17,\"text\":\"        /var/lib/homelab-cluster-admin \\\\\"},{\"line\":18,\"text\":\"        /opt \\\\\"},{\"line\":19,\"text\":\"        /srv \\\\\"},{\"line\":20,\"text\":\"        /usr/local/share \\\\\"},{\"line\":21,\"text\":\"        -type f \\\\\"},{\"line\":22,\"text\":\"        -path '*/.stage4i-control/activation.env' \\\\\"},{\"line\":23,\"text\":\"        -print 2>/dev/null |\"},{\"line\":24,\"text\":\"    sort -u\"},{\"line\":25,\"text\":\")\"},{\"line\":26,\"text\":\"\"},{\"line\":27,\"text\":\"if [ \\\"${#ACTIVATION_FILES[@]}\\\" -ne 1 ]; then\"},{\"line\":28,\"text\":\"    echo \\\"STAGE4I_VERIFICATION_STATUS=FAILED\\\"\"},{\"line\":29,\"text\":\"    echo \\\"REASON=ACTIVATION_FILE_COUNT_${#ACTIVATION_FILES[@]}\\\"\"},{\"line\":30,\"text\":\"    exit 50\"},{\"line\":31,\"text\":\"fi\"},{\"line\":32,\"text\":\"\"},{\"line\":33,\"text\":\"ACTIVATION_FILE=\\\"${ACTIVATION_FILES[0]}\\\"\"},{\"line\":34,\"text\":\"CONTROL_DIR=\\\"$(dirname \\\"$ACTIVATION_FILE\\\")\\\"\"},{\"line\":35,\"text\":\"ACTIVE_LINK=\\\"$CONTROL_DIR/active\\\"\"},{\"line\":36,\"text\":\"ROLLBACK_LINK=\\\"$CONTROL_DIR/rollback\\\"\"},{\"line\":37,\"text\":\"\"},{\"line\":38,\"text\":\"# shellcheck disable=SC1090\"},{\"line\":39,\"text\":\"source \\\"$ACTIVATION_FILE\\\"\"},{\"line\":40,\"text\":\"\"},{\"line\":41,\"text\":\"test \\\"${STAGE4I_ACTIVATION_STATUS:-}\\\" = \\\"ACTIVE\\\"\"},{\"line\":42,\"text\":\"test -L \\\"$ACTIVE_LINK\\\"\"},{\"line\":43,\"text\":\"test -L \\\"$ROLLBACK_LINK\\\"\"},{\"line\":44,\"text\":\"\"},{\"line\":45,\"text\":\"ACTIVE_RESOLVED=\\\"$(readlink -f \\\"$ACTIVE_LINK\\\")\\\"\"},{\"line\":46,\"text\":\"ROLLBACK_RESOLVED=\\\"$(readlink -f \\\"$ROLLBACK_LINK\\\")\\\"\"},{\"line\":47,\"text\":\"\"},{\"line\":48,\"text\":\"test \\\"$ACTIVE_RESOLVED\\\" = \\\"${ACTIVE_PATH:-}\\\"\"},{\"line\":49,\"text\":\"test \\\"$ROLLBACK_RESOLVED\\\" = \\\"${ROLLBACK_PATH:-}\\\"\"},{\"line\":50,\"text\":\"test -d \\\"$ACTIVE_RESOLVED\\\"\"},{\"line\":51,\"text\":\"test -d \\\"$ROLLBACK_RESOLVED\\\"\"},{\"line\":52,\"text\":\"\"},{\"line\":53,\"text\":\"META=\\\"$ACTIVE_RESOLVED/IMMUTABLE.env\\\"\"},{\"line\":54,\"text\":\"MANIFEST=\\\"$ACTIVE_RESOLVED/MANIFEST.sha256\\\"\"},{\"line\":55,\"text\":\"\"},{\"line\":56,\"text\":\"test -f \\\"$META\\\"\"},{\"line\":57,\"text\":\"test -f \\\"$MANIFEST\\\"\"},{\"line\":58,\"text\":\"grep -q \\\"^BUILD_MARKER=$MARKER$\\\" \\\"$META\\\"\"},{\"line\":59,\"text\":\"grep -q '^TASK_STAGE=4I$' \\\"$META\\\"\"},{\"line\":60,\"text\":\"grep -q '^TASK_STATUS=IMMUTABLE$' \\\"$META\\\"\"},{\"line\":61,\"text\":\"\"},{\"line\":62,\"text\":\"(\"},{\"line\":63,\"text\":\"    cd \\\"$ACTIVE_RESOLVED\\\"\"},{\"line\":64,\"text\":\"    sha256sum -c MANIFEST.sha256 >/dev/null\"},{\"line\":65,\"text\":\")\"},{\"line\":66,\"text\":\"\"},{\"line\":67,\"text\":\"if find \\\"$ACTIVE_RESOLVED\\\" \\\\\"},{\"line\":68,\"text\":\"    \\\\( ! -user root -o ! -group root \\\\) \\\\\"},{\"line\":69,\"text\":\"    -print -quit |\"},{\"line\":70,\"text\":\"    grep -q .\"},{\"line\":71,\"text\":\"then\"},{\"line\":72,\"text\":\"    echo \\\"STAGE4I_VERIFICATION_STATUS=FAILED\\\"\"},{\"line\":73,\"text\":\"    echo \\\"REASON=INVALID_OWNER\\\"\"},{\"line\":74,\"text\":\"    exit 51\"},{\"line\":75,\"text\":\"fi\"},{\"line\":76,\"text\":\"\"},{\"line\":77,\"text\":\"if find \\\"$ACTIVE_RESOLVED\\\" \\\\\"},{\"line\":78,\"text\":\"    -perm /0222 \\\\\"},{\"line\":79,\"text\":\"    -print -quit |\"},{\"line\":80,\"text\":\"    grep -q .\"},{\"line\":81,\"text\":\"then\"},{\"line\":82,\"text\":\"    echo \\\"STAGE4I_VERIFICATION_STATUS=FAILED\\\"\"},{\"line\":83,\"text\":\"    echo \\\"REASON=WRITABLE_IMMUTABLE_CONTENT\\\"\"},{\"line\":84,\"text\":\"    exit 52\"},{\"line\":85,\"text\":\"fi\"},{\"line\":86,\"text\":\"\"},{\"line\":87,\"text\":\"install -d -m 0750 -o root -g root \\\"$STATE_DIR\\\"\"},{\"line\":88,\"text\":\"\"},{\"line\":89,\"text\":\"{\"},{\"line\":90,\"text\":\"    echo \\\"STAGE4I_VERIFICATION_STATUS=SUCCESS\\\"\"},{\"line\":91,\"text\":\"    echo \\\"VERIFIED_AT_UTC=$(date -u +%Y-%m-%dT%H:%M:%SZ)\\\"\"},{\"line\":92,\"text\":\"    echo \\\"ACTIVE_VERSION=${ACTIVE_VERSION:-UNKNOWN}\\\"\"},{\"line\":93,\"text\":\"    echo \\\"ACTIVE_PATH=$ACTIVE_RESOLVED\\\"\"},{\"line\":94,\"text\":\"    echo \\\"ROLLBACK_VERSION=${ROLLBACK_VERSION:-UNKNOWN}\\\"\"},{\"line\":95,\"text\":\"    echo \\\"ROLLBACK_PATH=$ROLLBACK_RESOLVED\\\"\"},{\"line\":96,\"text\":\"    echo \\\"MANIFEST_SHA256=$(sha256sum \\\"$MANIFEST\\\" | awk '{print $1}')\\\"\"},{\"line\":97,\"text\":\"    echo \\\"VERIFIED_ON_HOST=$(hostname -f 2>/dev/null || hostname)\\\"\"},{\"line\":98,\"text\":\"} >\\\"$TMP\\\"\"},{\"line\":99,\"text\":\"\"},{\"line\":100,\"text\":\"chown root:root \\\"$TMP\\\"\"},{\"line\":101,\"text\":\"chmod 0440 \\\"$TMP\\\"\"},{\"line\":102,\"text\":\"mv -f \\\"$TMP\\\" \\\"$REPORT\\\"\"},{\"line\":103,\"text\":\"trap - EXIT\"},{\"line\":104,\"text\":\"\"},{\"line\":105,\"text\":\"cat \\\"$REPORT\\\"\"}]},{\"bytes\":616,\"line_count\":19,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"priority\":90,\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"Requires=postgresql.service\"},{\"line\":6,\"text\":\"\"},{\"line\":7,\"text\":\"[Service]\"},{\"line\":8,\"text\":\"Type=oneshot\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":13,\"text\":\"UMask=0027\"},{\"line\":14,\"text\":\"NoNewPrivileges=true\"},{\"line\":15,\"text\":\"PrivateTmp=true\"},{\"line\":16,\"text\":\"ProtectSystem=strict\"},{\"line\":17,\"text\":\"ProtectHome=true\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}]},{\"bytes\":231,\"line_count\":12,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"priority\":90,\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=30s\"},{\"line\":6,\"text\":\"OnUnitActiveSec=60s\"},{\"line\":7,\"text\":\"AccuracySec=10s\"},{\"line\":8,\"text\":\"Persistent=true\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"},{\"line\":10,\"text\":\"\"},{\"line\":11,\"text\":\"[Install]\"},{\"line\":12,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":803,\"line_count\":28,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"priority\":90,\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"Requires=postgresql.service\"},{\"line\":6,\"text\":\"\"},{\"line\":7,\"text\":\"[Service]\"},{\"line\":8,\"text\":\"Type=simple\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":13,\"text\":\"ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\"},{\"line\":14,\"text\":\"Restart=on-failure\"},{\"line\":15,\"text\":\"RestartSec=5\"},{\"line\":16,\"text\":\"UMask=0027\"},{\"line\":17,\"text\":\"NoNewPrivileges=true\"},{\"line\":18,\"text\":\"PrivateTmp=true\"},{\"line\":19,\"text\":\"ProtectSystem=strict\"},{\"line\":20,\"text\":\"ProtectHome=true\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"},{\"line\":23,\"text\":\"RestrictSUIDSGID=true\"},{\"line\":24,\"text\":\"LockPersonality=true\"},{\"line\":25,\"text\":\"RestrictRealtime=true\"},{\"line\":26,\"text\":\"\"},{\"line\":27,\"text\":\"[Install]\"},{\"line\":28,\"text\":\"WantedBy=multi-user.target\"}]},{\"bytes\":1190,\"line_count\":42,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"priority\":90,\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":3,\"text\":\"After=network-online.target\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"\"},{\"line\":6,\"text\":\"[Service]\"},{\"line\":7,\"text\":\"Type=oneshot\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":12,\"text\":\"UMask=0077\"},{\"line\":13,\"text\":\"NoNewPrivileges=yes\"},{\"line\":14,\"text\":\"PrivateTmp=yes\"},{\"line\":15,\"text\":\"PrivateDevices=yes\"},{\"line\":16,\"text\":\"ProtectSystem=strict\"},{\"line\":17,\"text\":\"ProtectHome=yes\"},{\"line\":18,\"text\":\"ProtectKernelTunables=yes\"},{\"line\":19,\"text\":\"ProtectKernelModules=yes\"},{\"line\":20,\"text\":\"ProtectKernelLogs=yes\"},{\"line\":21,\"text\":\"ProtectControlGroups=yes\"},{\"line\":22,\"text\":\"ProtectClock=yes\"},{\"line\":23,\"text\":\"ProtectHostname=yes\"},{\"line\":24,\"text\":\"LockPersonality=yes\"},{\"line\":25,\"text\":\"MemoryDenyWriteExecute=yes\"},{\"line\":26,\"text\":\"RestrictSUIDSGID=yes\"},{\"line\":27,\"text\":\"RestrictRealtime=yes\"},{\"line\":28,\"text\":\"RemoveIPC=yes\"},{\"line\":29,\"text\":\"CapabilityBoundingSet=\"},{\"line\":30,\"text\":\"AmbientCapabilities=\"},{\"line\":31,\"text\":\"RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\"},{\"line\":32,\"text\":\"IPAddressDeny=any\"},{\"line\":33,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":34,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":35,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":36,\"text\":\"IPAddressAllow=[PRIVATE_IP]\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":40,\"text\":\"StateDirectoryMode=0750\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"},{\"line\":42,\"text\":\"TimeoutStartSec=70\"}]},{\"bytes\":223,\"line_count\":12,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"priority\":90,\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=2min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=60s\"},{\"line\":7,\"text\":\"RandomizedDelaySec=5s\"},{\"line\":8,\"text\":\"Persistent=true\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"},{\"line\":10,\"text\":\"\"},{\"line\":11,\"text\":\"[Install]\"},{\"line\":12,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":133,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"priority\":90,\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}]},{\"bytes\":154,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"priority\":90,\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=5min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=5min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":131,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"priority\":90,\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}]},{\"bytes\":149,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"priority\":90,\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=3min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=5min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":127,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"priority\":90,\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}]},{\"bytes\":151,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"priority\":90,\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=2min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=5min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":138,\"line_count\":6,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"priority\":90,\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Service]\"},{\"line\":5,\"text\":\"Type=oneshot\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}]},{\"bytes\":154,\"line_count\":10,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"priority\":90,\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=1min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=1min\"},{\"line\":7,\"text\":\"Persistent=true\"},{\"line\":8,\"text\":\"\"},{\"line\":9,\"text\":\"[Install]\"},{\"line\":10,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":301,\"line_count\":13,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"priority\":90,\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":3,\"text\":\"After=network-online.target\"},{\"line\":4,\"text\":\"Wants=network-online.target\"},{\"line\":5,\"text\":\"\"},{\"line\":6,\"text\":\"[Service]\"},{\"line\":7,\"text\":\"Type=simple\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"},{\"line\":9,\"text\":\"Restart=always\"},{\"line\":10,\"text\":\"RestartSec=5\"},{\"line\":11,\"text\":\"\"},{\"line\":12,\"text\":\"[Install]\"},{\"line\":13,\"text\":\"WantedBy=multi-user.target\"}]},{\"bytes\":465,\"line_count\":22,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"priority\":90,\"sha256\":\"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Verify active immutable Stage 4I task version\"},{\"line\":3,\"text\":\"After=local-fs.target\"},{\"line\":4,\"text\":\"\"},{\"line\":5,\"text\":\"[Service]\"},{\"line\":6,\"text\":\"Type=oneshot\"},{\"line\":7,\"text\":\"User=root\"},{\"line\":8,\"text\":\"Group=root\"},{\"line\":9,\"text\":\"ExecStart=/usr/local/sbin/homelab-stage4i-verify\"},{\"line\":10,\"text\":\"NoNewPrivileges=true\"},{\"line\":11,\"text\":\"PrivateDevices=true\"},{\"line\":12,\"text\":\"ProtectClock=true\"},{\"line\":13,\"text\":\"ProtectControlGroups=true\"},{\"line\":14,\"text\":\"ProtectHome=true\"},{\"line\":15,\"text\":\"ProtectKernelLogs=true\"},{\"line\":16,\"text\":\"ProtectKernelModules=true\"},{\"line\":17,\"text\":\"ProtectKernelTunables=true\"},{\"line\":18,\"text\":\"RestrictNamespaces=true\"},{\"line\":19,\"text\":\"RestrictRealtime=true\"},{\"line\":20,\"text\":\"LockPersonality=true\"},{\"line\":21,\"text\":\"MemoryDenyWriteExecute=true\"},{\"line\":22,\"text\":\"UMask=0027\"}]},{\"bytes\":224,\"line_count\":12,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"priority\":90,\"sha256\":\"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396\",\"source\":[{\"line\":1,\"text\":\"[Unit]\"},{\"line\":2,\"text\":\"Description=Regularly verify active immutable Stage 4I task version\"},{\"line\":3,\"text\":\"\"},{\"line\":4,\"text\":\"[Timer]\"},{\"line\":5,\"text\":\"OnBootSec=2min\"},{\"line\":6,\"text\":\"OnUnitActiveSec=15min\"},{\"line\":7,\"text\":\"AccuracySec=1min\"},{\"line\":8,\"text\":\"Persistent=true\"},{\"line\":9,\"text\":\"Unit=homelab-stage4i-verify.service\"},{\"line\":10,\"text\":\"\"},{\"line\":11,\"text\":\"[Install]\"},{\"line\":12,\"text\":\"WantedBy=timers.target\"}]},{\"bytes\":380,\"line_count\":1,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"priority\":70,\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\",\"source\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}]},{\"bytes\":217,\"line_count\":1,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"priority\":70,\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\",\"source\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}]},{\"bytes\":128,\"line_count\":1,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"priority\":70,\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\",\"source\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md\"}]},{\"bytes\":1535,\"line_count\":1,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"priority\":70,\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\",\"source\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}]},{\"bytes\":89,\"line_count\":3,\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"priority\":50,\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\",\"source\":[{\"line\":1,\"text\":\"[Service]\"},{\"line\":2,\"text\":\"ExecStart=\"},{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}]},{\"bytes\":28599,\"line_count\":484,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"priority\":50,\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\",\"source\":[{\"line\":1,\"text\":\"<REDACTED_SECRET_LINE>\"},{\"line\":2,\"text\":\"from collections import defaultdict\"},{\"line\":3,\"text\":\"from contextlib import contextmanager\"},{\"line\":4,\"text\":\"from datetime import datetime\"},{\"line\":5,\"text\":\"from urllib.parse import parse_qs\"},{\"line\":6,\"text\":\"import psycopg2\"},{\"line\":7,\"text\":\"from psycopg2.extras import RealDictCursor\"},{\"line\":8,\"text\":\"from fastapi import FastAPI, HTTPException, Request\"},{\"line\":9,\"text\":\"from fastapi.responses import HTMLResponse, RedirectResponse\"},{\"line\":10,\"text\":\"from starlette.middleware.trustedhost import TrustedHostMiddleware\"},{\"line\":11,\"text\":\"\"},{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =<REDACTED>\"},{\"line\":14,\"text\":\"COOKIE =<REDACTED>\"},{\"line\":15,\"text\":\"failures, locks = defaultdict(list), {}\"},{\"line\":16,\"text\":\"app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":18,\"text\":\"\"},{\"line\":19,\"text\":\"def db():\"},{\"line\":20,\"text\":\"    return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":21,\"text\":\"\"},{\"line\":22,\"text\":\"def enc(value):\"},{\"line\":23,\"text\":\"    return base64.urlsafe_b64encode(value).rstrip(b\\\"=\\\").decode()\"},{\"line\":24,\"text\":\"\"},{\"line\":25,\"text\":\"def dec(value):\"},{\"line\":26,\"text\":\"    return base64.urlsafe_b64decode(value + \\\"=\\\" * (-len(value) % 4))\"},{\"line\":27,\"text\":\"\"},{\"line\":28,\"text\":\"def sign(value):\"},{\"line\":29,\"text\":\"    return enc(hmac.new(KEY, value.encode(), hashlib.sha256).digest())\"},{\"line\":30,\"text\":\"\"},{\"line\":31,\"text\":\"def ip(request):\"},{\"line\":32,\"text\":\"    peer = request.client.host if request.client else \\\"unknown\\\"\"},{\"line\":33,\"text\":\"    if peer in {\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"::1\\\"}:\"},{\"line\":34,\"text\":\"        return request.headers.get(\\\"x-forwarded-for\\\", peer).split(\\\",\\\")[0].strip()\"},{\"line\":35,\"text\":\"    return peer\"},{\"line\":36,\"text\":\"\"},{\"line\":37,\"text\":\"def login_token(request): <REDACTED>\"},{\"line\":38,\"text\":\"    raw = f\\\"{int(time.time())}:{ip(request)}\\\"\"},{\"line\":39,\"text\":\"    return raw + \\\".\\\" + sign(raw)\"},{\"line\":40,\"text\":\"\"},{\"line\":41,\"text\":\"def valid_login_token(request, token): <REDACTED>\"},{\"line\":42,\"text\":\"    try:\"},{\"line\":43,\"text\":\"        raw, signature =<REDACTED>\"},{\"line\":44,\"text\":\"        stamp, address = raw.split(\\\":\\\", 1)\"},{\"line\":45,\"text\":\"        return hmac.compare_digest(signature, sign(raw)) and address == ip(request) and 0 <= time.time() - int(stamp) <= 600\"},{\"line\":46,\"text\":\"    except Exception:\"},{\"line\":47,\"text\":\"        return False\"},{\"line\":48,\"text\":\"\"},{\"line\":49,\"text\":\"def session_cookie(request): <REDACTED>\"},{\"line\":50,\"text\":\"    data =<REDACTED>\"},{\"line\":51,\"text\":\"    raw = enc(json.dumps(data,separators=(\\\",\\\",\\\":\\\")).encode())\"},{\"line\":52,\"text\":\"    return raw + \\\".\\\" + sign(raw)\"},{\"line\":53,\"text\":\"\"},{\"line\":54,\"text\":\"def session(request):\"},{\"line\":55,\"text\":\"    try:\"},{\"line\":56,\"text\":\"        raw, signature =<REDACTED>\"},{\"line\":57,\"text\":\"        data = json.loads(dec(raw))\"},{\"line\":58,\"text\":\"        expected = hashlib.sha256(request.headers.get(\\\"user-agent\\\",\\\"\\\").encode()).hexdigest()[:24]\"},{\"line\":59,\"text\":\"        return data if hmac.compare_digest(signature,sign(raw)) and data[\\\"exp\\\"] >= time.time() and data[\\\"ua\\\"] == expected else None\"},{\"line\":60,\"text\":\"    except Exception:\"},{\"line\":61,\"text\":\"        return None\"},{\"line\":62,\"text\":\"\"},{\"line\":63,\"text\":\"def password_ok(password): <REDACTED>\"},{\"line\":64,\"text\":\"    try:\"},{\"line\":65,\"text\":\"        scheme, rounds, salt, expected = open(PASSFILE,encoding=\\\"utf-8\\\").read().strip().split(\\\"$\\\",3)\"},{\"line\":66,\"text\":\"        actual =<REDACTED>\"},{\"line\":67,\"text\":\"        return scheme == \\\"pbkdf2_sha256\\\" and hmac.compare_digest(base64.urlsafe_b64encode(actual).decode(),expected)\"},{\"line\":68,\"text\":\"    except Exception:\"},{\"line\":69,\"text\":\"        return False\"},{\"line\":70,\"text\":\"\"},{\"line\":71,\"text\":\"def login_page(request, error=\\\"\\\"):\"},{\"line\":72,\"text\":\"    token =<REDACTED>\"},{\"line\":73,\"text\":\"    return f\\\"\\\"\\\"<!doctype html><html lang=<REDACTED>\"},{\"line\":74,\"text\":\"\"},{\"line\":75,\"text\":\"def status_kind(value):\"},{\"line\":76,\"text\":\"    text = \\\"UNKNOWN\\\" if value is None or str(value).strip() == \\\"\\\" else str(value).strip().upper()\"},{\"line\":77,\"text\":\"    if text in {\\\"OK\\\", \\\"ACTIVE\\\", \\\"RUNNING\\\", \\\"UP\\\", \\\"ONLINE\\\", \\\"HEALTHY\\\", \\\"RECOVERED\\\", \\\"CLOSED\\\", \\\"SUCCESS\\\", \\\"ENABLED\\\", \\\"READY\\\"}:\"},{\"line\":78,\"text\":\"        return \\\"ok\\\"\"},{\"line\":79,\"text\":\"    if text in {\\\"WARN\\\", \\\"WARNING\\\", \\\"PENDING\\\", \\\"DEGRADED\\\", \\\"ATTENTION\\\", \\\"STALE\\\"}:\"},{\"line\":80,\"text\":\"        return \\\"warn\\\"\"},{\"line\":81,\"text\":\"    if text in {\\\"OPEN\\\", \\\"ERROR\\\", \\\"FAILED\\\", \\\"DOWN\\\", \\\"OFFLINE\\\", \\\"CRITICAL\\\", \\\"P0\\\", \\\"P1\\\", \\\"FIRING\\\", \\\"UNHEALTHY\\\"}:\"},{\"line\":82,\"text\":\"        return \\\"bad\\\"\"},{\"line\":83,\"text\":\"    return \\\"unknown\\\"\"},{\"line\":84,\"text\":\"\"},{\"line\":85,\"text\":\"def status_badge(value):\"},{\"line\":86,\"text\":\"    text = \\\"UNKNOWN\\\" if value is None or str(value).strip() == \\\"\\\" else str(value)\"},{\"line\":87,\"text\":\"    kind = status_kind(text)\"},{\"line\":88,\"text\":\"    return \\\"<span class=\\\\\\\"state state-\\\" + kind + \\\"\\\\\\\"><span class=\\\\\\\"state-dot\\\\\\\"></span>\\\" + html.escape(text) + \\\"</span>\\\"\"},{\"line\":89,\"text\":\"\"},{\"line\":90,\"text\":\"def summary_card(title, value, state):\"},{\"line\":91,\"text\":\"    return \\\"<div class=\\\\\\\"card\\\\\\\"><div class=\\\\\\\"card-health\\\\\\\">\\\" + status_badge(state) + \\\"</div><b>\\\" + html.escape(str(value)) + \\\"</b><span>\\\" + html.escape(title) + \\\"</span></div>\\\"\"},{\"line\":92,\"text\":\"\"},{\"line\":93,\"text\":\"def aggregate_state(rows, keys=(\\\"status\\\",), empty_state=\\\"WARN\\\"):\"},{\"line\":94,\"text\":\"    if not rows:\"},{\"line\":95,\"text\":\"        return empty_state\"},{\"line\":96,\"text\":\"    kinds = []\"},{\"line\":97,\"text\":\"    for row in rows:\"},{\"line\":98,\"text\":\"        for key in keys:\"},{\"line\":99,\"text\":\"            if row.get(key) is not None:\"},{\"line\":100,\"text\":\"                kinds.append(status_kind(row.get(key)))\"},{\"line\":101,\"text\":\"    if \\\"bad\\\" in kinds:\"},{\"line\":102,\"text\":\"        return \\\"ERROR\\\"\"},{\"line\":103,\"text\":\"    if \\\"warn\\\" in kinds or \\\"unknown\\\" in kinds or not kinds:\"},{\"line\":104,\"text\":\"        return \\\"WARN\\\"\"},{\"line\":105,\"text\":\"    return \\\"OK\\\"\"},{\"line\":106,\"text\":\"\"},{\"line\":107,\"text\":\"def incident_section_state(rows):\"},{\"line\":108,\"text\":\"    if not rows:\"},{\"line\":109,\"text\":\"        return \\\"OK\\\"\"},{\"line\":110,\"text\":\"    severe = {\\\"P0\\\", \\\"P1\\\", \\\"CRITICAL\\\", \\\"ERROR\\\"}\"},{\"line\":111,\"text\":\"    for row in rows:\"},{\"line\":112,\"text\":\"        severity = str(row.get(\\\"severity\\\") or \\\"\\\").strip().upper()\"},{\"line\":113,\"text\":\"        state = row.get(\\\"status\\\")\"},{\"line\":114,\"text\":\"        if severity in severe or status_kind(state) == \\\"bad\\\":\"},{\"line\":115,\"text\":\"            return \\\"ERROR\\\"\"},{\"line\":116,\"text\":\"    return \\\"WARN\\\"\"},{\"line\":117,\"text\":\"\"},{\"line\":118,\"text\":\"# Все текущие и будущие подробные блоки dashboard должны использовать этот компонент.\"},{\"line\":119,\"text\":\"def health_section(title, count, state, content, wide=False):\"},{\"line\":120,\"text\":\"    kind = status_kind(state)\"},{\"line\":121,\"text\":\"    css_class = \\\"panel health-section\\\" + (\\\" wide\\\" if wide else \\\"\\\")\"},{\"line\":122,\"text\":\"    return \\\"<details class=\\\\\\\"\\\" + css_class + \\\"\\\\\\\"><summary><span class=\\\\\\\"section-heading\\\\\\\"><span class=\\\\\\\"state state-\\\" + kind + \\\"\\\\\\\" aria-label=\\\\\\\"\\\" + html.escape(str(state)) + \\\"\\\\\\\"><span class=\\\\\\\"state-dot\\\\\\\"></span></span><span class=\\\\\\\"section-title\\\\\\\">\\\" + html.escape(str(title)) + \\\"</span><span class=\\\\\\\"section-count\\\\\\\">\\\" + html.escape(str(count)) + \\\"</span></span><span class=\\\\\\\"section-chevron\\\\\\\" aria-hidden=\\\\\\\"true\\\\\\\">⌄</span></summary><div class=\\\\\\\"section-body\\\\\\\">\\\" + content + \\\"</div></details>\\\"\"},{\"line\":123,\"text\":\"\"},{\"line\":124,\"text\":\"def table(rows, columns):\"},{\"line\":125,\"text\":\"    if not rows:\"},{\"line\":126,\"text\":\"        return \\\"<p class=\\\\\\\"muted\\\\\\\">Данные ещё не собраны.</p>\\\"\"},{\"line\":127,\"text\":\"    head = \\\"\\\".join(\\\"<th>\\\" + html.escape(str(title)) + \\\"</th>\\\" for title, key in columns)\"},{\"line\":128,\"text\":\"    body = []\"},{\"line\":129,\"text\":\"    for row in rows:\"},{\"line\":130,\"text\":\"        cells = []\"},{\"line\":131,\"text\":\"        for title, key in columns:\"},{\"line\":132,\"text\":\"            value = row.get(key)\"},{\"line\":133,\"text\":\"            rendered = status_badge(value) if key in {\\\"status\\\", \\\"severity\\\"} else html.escape(str(value if value is not None else chr(8212)))\"},{\"line\":134,\"text\":\"            cells.append(\\\"<td>\\\" + rendered + \\\"</td>\\\")\"},{\"line\":135,\"text\":\"        row_kind = status_kind(row.get(\\\"status\\\", row.get(\\\"severity\\\", \\\"UNKNOWN\\\")))\"},{\"line\":136,\"text\":\"        body.append(\\\"<tr class=\\\\\\\"row-\\\" + row_kind + \\\"\\\\\\\">\\\" + \\\"\\\".join(cells) + \\\"</tr>\\\")\"},{\"line\":137,\"text\":\"    return \\\"<table><thead><tr>\\\" + head + \\\"</tr></thead><tbody>\\\" + \\\"\\\".join(body) + \\\"</tbody></table>\\\"\"},{\"line\":138,\"text\":\"\"},{\"line\":139,\"text\":\"@app.middleware(\\\"http\\\")\"},{\"line\":140,\"text\":\"async def headers(request, call_next):\"},{\"line\":141,\"text\":\"    response = await call_next(request)\"},{\"line\":142,\"text\":\"    q = chr(39)\"},{\"line\":143,\"text\":\"    response.headers[\\\"Content-Security-Policy\\\"] = f\\\"default-src {q}self{q};style-src {q}self{q} {q}unsafe-inline{q};frame-ancestors {q}none{q};base-uri {q}none{q};form-action {q}self{q}\\\"\"},{\"line\":144,\"text\":\"    response.headers[\\\"X-Frame-Options\\\"] = \\\"DENY\\\"\"},{\"line\":145,\"text\":\"    response.headers[\\\"X-Content-Type-Options\\\"] = \\\"nosniff\\\"\"},{\"line\":146,\"text\":\"    response.headers[\\\"Referrer-Policy\\\"] = \\\"no-referrer\\\"\"},{\"line\":147,\"text\":\"    response.headers[\\\"Cache-Control\\\"] = \\\"no-store\\\"\"},{\"line\":148,\"text\":\"    return response\"},{\"line\":149,\"text\":\"\"},{\"line\":150,\"text\":\"@app.get(\\\"/health\\\")\"},{\"line\":151,\"text\":\"def health():\"},{\"line\":152,\"text\":\"    return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":153,\"text\":\"\"},{\"line\":154,\"text\":\"@app.get(\\\"/login\\\",response_class=HTMLResponse)\"},{\"line\":155,\"text\":\"def get_login(request:Request):\"},{\"line\":156,\"text\":\"    return RedirectResponse(\\\"/\\\",303) if session(request) else HTMLResponse(login_page(request))\"},{\"line\":157,\"text\":\"\"},{\"line\":158,\"text\":\"@app.post(\\\"/login\\\",response_class=HTMLResponse)\"},{\"line\":159,\"text\":\"async def post_login(request:Request):\"},{\"line\":160,\"text\":\"    address, now = ip(request), time.time()\"},{\"line\":161,\"text\":\"    failures[address] = [stamp for stamp in failures[address] if now-stamp < 900]\"},{\"line\":162,\"text\":\"    if locks.get(address,0) > now:\"},{\"line\":163,\"text\":\"        return HTMLResponse(login_page(request,\\\"Слишком много попыток. Повторите позже.\\\"),429)\"},{\"line\":164,\"text\":\"    if int(request.headers.get(\\\"content-length\\\",\\\"0\\\") or 0) > 4096:\"},{\"line\":165,\"text\":\"        raise HTTPException(413)\"},{\"line\":166,\"text\":\"    form = parse_qs((await request.body()).decode(\\\"utf-8\\\",\\\"replace\\\"))\"},{\"line\":167,\"text\":\"    if not valid_login_token(request,form.get(\\\"csrf\\\",[\\\"\\\"])[0]) or not password_ok(form.get(\\\"password\\\",[\\\"\\\"])[0]): <REDACTED>\"},{\"line\":168,\"text\":\"        failures[address].append(now)\"},{\"line\":169,\"text\":\"        if len(failures[address]) >= 5:\"},{\"line\":170,\"text\":\"            locks[address], failures[address] = now+900, []\"},{\"line\":171,\"text\":\"        return HTMLResponse(login_page(request,\\\"Неверный пароль.\\\"),401)\"},{\"line\":172,\"text\":\"    failures.pop(address,None); locks.pop(address,None)\"},{\"line\":173,\"text\":\"    response = RedirectResponse(\\\"/\\\",303)\"},{\"line\":174,\"text\":\"    response.set_cookie(COOKIE,session_cookie(request),max_age=<REDACTED>\"},{\"line\":175,\"text\":\"    return response\"},{\"line\":176,\"text\":\"\"},{\"line\":177,\"text\":\"@app.post(\\\"/logout\\\")\"},{\"line\":178,\"text\":\"async def logout(request:Request):\"},{\"line\":179,\"text\":\"    data = session(request)\"},{\"line\":180,\"text\":\"    if not data:\"},{\"line\":181,\"text\":\"        raise HTTPException(401)\"},{\"line\":182,\"text\":\"    form = parse_qs((await request.body()).decode(\\\"utf-8\\\",\\\"replace\\\"))\"},{\"line\":183,\"text\":\"    if not hmac.compare_digest(form.get(\\\"csrf\\\",[\\\"\\\"])[0],data[\\\"csrf\\\"]):\"},{\"line\":184,\"text\":\"        raise HTTPException(403)\"},{\"line\":185,\"text\":\"    response = RedirectResponse(\\\"/login\\\",303)\"},{\"line\":186,\"text\":\"    response.delete_cookie(COOKIE,path=<REDACTED>\"},{\"line\":187,\"text\":\"    return response\"},{\"line\":188,\"text\":\"\"},{\"line\":189,\"text\":\"@app.get(\\\"/api/summary\\\")\"},{\"line\":190,\"text\":\"def api_summary(request:Request):\"},{\"line\":191,\"text\":\"    if not session(request):\"},{\"line\":192,\"text\":\"        raise HTTPException(401)\"},{\"line\":193,\"text\":\"    with db() as connection, connection.cursor(cursor_factory=RealDictCursor) as cursor:\"},{\"line\":194,\"text\":\"        cursor.execute(\\\"SELECT (SELECT count(*) FROM nodes) nodes,(SELECT count(*) FROM guests) guests,(SELECT count(*) FROM services) services,(SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)) incidents\\\",(\\\"RECOVERED\\\",\\\"CLOSED\\\"))\"},{\"line\":195,\"text\":\"        return dict(cursor.fetchone())\"},{\"line\":196,\"text\":\"\"},{\"line\":197,\"text\":\"@app.get(\\\"/\\\",response_class=HTMLResponse)\"},{\"line\":198,\"text\":\"def dashboard(request:Request):\"},{\"line\":199,\"text\":\"    data = session(request)\"},{\"line\":200,\"text\":\"    if not data:\"},{\"line\":201,\"text\":\"        return RedirectResponse(\\\"/login\\\",303)\"},{\"line\":202,\"text\":\"    with db() as connection, connection.cursor(cursor_factory=RealDictCursor) as cursor:\"},{\"line\":203,\"text\":\"        cursor.execute(\\\"SELECT (SELECT count(*) FROM nodes) nodes,(SELECT count(*) FROM guests) guests,(SELECT count(*) FROM services) services,(SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)) incidents\\\",(\\\"RECOVERED\\\",\\\"CLOSED\\\"))\"},{\"line\":204,\"text\":\"        counts = dict(cursor.fetchone())\"},{\"line\":205,\"text\":\"        cursor.execute(\\\"SELECT name,role,status,last_seen FROM nodes ORDER BY name\\\"); nodes=[dict(x) for x in cursor.fetchall()]\"},{\"line\":206,\"text\":\"        cursor.execute(\\\"SELECT name,scope,host_name,container_name,status,last_seen FROM services ORDER BY name LIMIT 250\\\"); services=[dict(x) for x in cursor.fetchall()]\"},{\"line\":207,\"text\":\"        cursor.execute(\\\"SELECT severity,title,status,last_seen FROM incidents WHERE status NOT IN (%s,%s) ORDER BY last_seen DESC LIMIT 100\\\",(\\\"RECOVERED\\\",\\\"CLOSED\\\")); incidents=[dict(x) for x in cursor.fetchall()]\"},{\"line\":208,\"text\":\"    cards = \\\"\\\".join([summary_card(\\\"Ноды\\\", counts[\\\"nodes\\\"], \\\"OK\\\" if counts[\\\"nodes\\\"] > 0 else \\\"ERROR\\\"), summary_card(\\\"VM и CT\\\", counts[\\\"guests\\\"], \\\"OK\\\" if counts[\\\"guests\\\"] > 0 else \\\"ERROR\\\"), summary_card(\\\"Сервисы\\\", counts[\\\"services\\\"], \\\"OK\\\" if counts[\\\"services\\\"] > 0 else \\\"ERROR\\\"), summary_card(\\\"Инциденты\\\", counts[\\\"incidents\\\"], \\\"ERROR\\\" if counts[\\\"incidents\\\"] > 0 else \\\"OK\\\")])\"},{\"line\":209,\"text\":\"    csrf = data[\\\"csrf\\\"]\"},{\"line\":210,\"text\":\"    nodes_table = table(nodes,[(\\\"Нода\\\",\\\"name\\\"),(\\\"Роль\\\",\\\"role\\\"),(\\\"Статус\\\",\\\"status\\\"),(\\\"Последняя связь\\\",\\\"last_seen\\\")])\"},{\"line\":211,\"text\":\"    incidents_table = table(incidents,[(\\\"Severity\\\",\\\"severity\\\"),(\\\"Инцидент\\\",\\\"title\\\"),(\\\"Статус\\\",\\\"status\\\"),(\\\"Обновлён\\\",\\\"last_seen\\\")])\"},{\"line\":212,\"text\":\"    services_table = table(services,[(\\\"Сервис\\\",\\\"name\\\"),(\\\"Scope\\\",\\\"scope\\\"),(\\\"Хост\\\",\\\"host_name\\\"),(\\\"Контейнер\\\",\\\"container_name\\\"),(\\\"Статус\\\",\\\"status\\\"),(\\\"Последняя связь\\\",\\\"last_seen\\\")])\"},{\"line\":213,\"text\":\"    nodes_state = aggregate_state(nodes, (\\\"status\\\",), \\\"ERROR\\\")\"},{\"line\":214,\"text\":\"    incidents_state = incident_section_state(incidents)\"},{\"line\":215,\"text\":\"    services_state = aggregate_state(services, (\\\"status\\\",), \\\"ERROR\\\")\"},{\"line\":216,\"text\":\"    nodes_html = health_section(\\\"Ноды\\\", len(nodes), nodes_state, nodes_table)\"},{\"line\":217,\"text\":\"    incidents_html = health_section(\\\"Активные инциденты\\\", len(incidents), incidents_state, incidents_table)\"},{\"line\":218,\"text\":\"    services_html = health_section(\\\"Сервисы\\\", len(services), services_state, services_table, wide=True)\"},{\"line\":219,\"text\":\"    engine_badge = status_badge(\\\"OK\\\")\"},{\"line\":220,\"text\":\"    page = f\\\"\\\"\\\"<!doctype html><html lang=\\\"ru\\\"><head><meta charset=\\\"utf-8\\\"><meta name=\\\"viewport\\\" content=\\\"width=device-width,initial-scale=1\\\"><meta http-equiv=\\\"refresh\\\" content=\\\"30\\\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\\\"post\\\" action=\\\"/logout\\\"><input type=\\\"hidden\\\" name=\\\"csrf\\\" value=\\\"{csrf}\\\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\\\"\\\"\\\"\"},{\"line\":221,\"text\":\"    return HTMLResponse(page)\"},{\"line\":222,\"text\":\"\"},{\"line\":223,\"text\":\"from fastapi.responses import PlainTextResponse\"},{\"line\":224,\"text\":\"\"},{\"line\":225,\"text\":\"@app.get(\\\"/metrics\\\", response_class=PlainTextResponse)\"},{\"line\":226,\"text\":\"def metrics():\"},{\"line\":227,\"text\":\"    with db() as connection, connection.cursor() as cursor:\"},{\"line\":228,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM nodes\\\")\"},{\"line\":229,\"text\":\"        nodes = cursor.fetchone()[0]\"},{\"line\":230,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM guests\\\")\"},{\"line\":231,\"text\":\"        guests = cursor.fetchone()[0]\"},{\"line\":232,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM services\\\")\"},{\"line\":233,\"text\":\"        services = cursor.fetchone()[0]\"},{\"line\":234,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM observations\\\")\"},{\"line\":235,\"text\":\"        observations = cursor.fetchone()[0]\"},{\"line\":236,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)\\\", (\\\"RECOVERED\\\",\\\"CLOSED\\\"))\"},{\"line\":237,\"text\":\"        incidents = cursor.fetchone()[0]\"},{\"line\":238,\"text\":\"        cursor.execute(\\\"SELECT COALESCE(EXTRACT(EPOCH FROM (now()-max(finished_at))),999999) FROM collector_runs WHERE status=%s\\\", (\\\"OK\\\",))\"},{\"line\":239,\"text\":\"        collector_age = float(cursor.fetchone()[0])\"},{\"line\":240,\"text\":\"        cursor.execute(\\\"SELECT severity,count(*) FROM incidents WHERE status NOT IN (%s,%s) GROUP BY severity ORDER BY severity\\\", (\\\"RECOVERED\\\",\\\"CLOSED\\\"))\"},{\"line\":241,\"text\":\"        severity_rows = cursor.fetchall()\"},{\"line\":242,\"text\":\"    lines = [\"},{\"line\":243,\"text\":\"        \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\"        \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\"        \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\"        \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\"        \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\"        \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\"        \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\"        \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\"        \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\"        \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\"        \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\"        \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\"        \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\"        \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\"        \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\"        \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\"        \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\"        \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\"        \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\"        \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\"        \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":264,\"text\":\"    ]\"},{\"line\":265,\"text\":\"    for severity, count in severity_rows:\"},{\"line\":266,\"text\":\"        clean = str(severity).replace(chr(34), \\\"\\\")\"},{\"line\":267,\"text\":\"        lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"},{\"line\":268,\"text\":\"    return PlainTextResponse(\\\"\\\\n\\\".join(lines) + \\\"\\\\n\\\", media_type=\\\"text/plain; version=0.0.4\\\")\"},{\"line\":269,\"text\":\"\"},{\"line\":270,\"text\":\"@contextmanager\"},{\"line\":271,\"text\":\"def api_v1_read_cursor():\"},{\"line\":272,\"text\":\"    connection = db()\"},{\"line\":273,\"text\":\"    connection.autocommit = True\"},{\"line\":274,\"text\":\"    cursor = connection.cursor(cursor_factory=RealDictCursor)\"},{\"line\":275,\"text\":\"    try:\"},{\"line\":276,\"text\":\"        cursor.execute(\\\"BEGIN READ ONLY\\\")\"},{\"line\":277,\"text\":\"        cursor.execute(\\\"SET LOCAL statement_timeout TO 5000\\\")\"},{\"line\":278,\"text\":\"        yield cursor\"},{\"line\":279,\"text\":\"    finally:\"},{\"line\":280,\"text\":\"        try:\"},{\"line\":281,\"text\":\"            cursor.execute(\\\"ROLLBACK\\\")\"},{\"line\":282,\"text\":\"        except Exception:\"},{\"line\":283,\"text\":\"            pass\"},{\"line\":284,\"text\":\"        cursor.close()\"},{\"line\":285,\"text\":\"        connection.close()\"},{\"line\":286,\"text\":\"\"},{\"line\":287,\"text\":\"\"},{\"line\":288,\"text\":\"def api_v1_require_session(request: Request):\"},{\"line\":289,\"text\":\"    data = session(request)\"},{\"line\":290,\"text\":\"    if not data:\"},{\"line\":291,\"text\":\"        raise HTTPException(401)\"},{\"line\":292,\"text\":\"    return data\"},{\"line\":293,\"text\":\"\"},{\"line\":294,\"text\":\"\"},{\"line\":295,\"text\":\"def api_v1_limit(value):\"},{\"line\":296,\"text\":\"    if value < 1 or value > 500:\"},{\"line\":297,\"text\":\"        raise HTTPException(422, detail=\\\"limit must be between 1 and 500\\\")\"},{\"line\":298,\"text\":\"    return value\"},{\"line\":299,\"text\":\"\"},{\"line\":300,\"text\":\"\"},{\"line\":301,\"text\":\"def api_v1_updated_since(value):\"},{\"line\":302,\"text\":\"    if not value:\"},{\"line\":303,\"text\":\"        return None\"},{\"line\":304,\"text\":\"    try:\"},{\"line\":305,\"text\":\"        return datetime.fromisoformat(value.replace(\\\"Z\\\", \\\"+00:00\\\"))\"},{\"line\":306,\"text\":\"    except Exception:\"},{\"line\":307,\"text\":\"        raise HTTPException(422, detail=\\\"updated_since must be ISO 8601\\\")\"},{\"line\":308,\"text\":\"\"},{\"line\":309,\"text\":\"\"},{\"line\":310,\"text\":\"def api_v1_encode_id_cursor(value):\"},{\"line\":311,\"text\":\"    return enc(str(int(value)).encode())\"},{\"line\":312,\"text\":\"\"},{\"line\":313,\"text\":\"\"},{\"line\":314,\"text\":\"def api_v1_decode_id_cursor(value):\"},{\"line\":315,\"text\":\"    if not value:\"},{\"line\":316,\"text\":\"        return None\"},{\"line\":317,\"text\":\"    try:\"},{\"line\":318,\"text\":\"        result = int(dec(value).decode())\"},{\"line\":319,\"text\":\"        if result < 0:\"},{\"line\":320,\"text\":\"            raise ValueError()\"},{\"line\":321,\"text\":\"        return result\"},{\"line\":322,\"text\":\"    except Exception:\"},{\"line\":323,\"text\":\"        raise HTTPException(422, detail=\\\"invalid cursor\\\")\"},{\"line\":324,\"text\":\"\"},{\"line\":325,\"text\":\"\"},{\"line\":326,\"text\":\"@app.get(\\\"/api/v1/entities\\\")\"},{\"line\":327,\"text\":\"def api_v1_entities(request: Request, limit: int = 100, cursor: str = \\\"\\\", entity_type: str = \\\"\\\", lifecycle_status: str = \\\"\\\", source_of_truth: str = \\\"\\\", updated_since: str = \\\"\\\", search: str = \\\"\\\"):\"},{\"line\":328,\"text\":\"    api_v1_require_session(request)\"},{\"line\":329,\"text\":\"    limit = api_v1_limit(limit)\"},{\"line\":330,\"text\":\"    clauses = []\"},{\"line\":331,\"text\":\"    params = []\"},{\"line\":332,\"text\":\"    if cursor:\"},{\"line\":333,\"text\":\"        clauses.append(\\\"e.entity_key > %s\\\")\"},{\"line\":334,\"text\":\"        params.append(cursor)\"},{\"line\":335,\"text\":\"    if entity_type:\"},{\"line\":336,\"text\":\"        clauses.append(\\\"e.entity_type = %s\\\")\"},{\"line\":337,\"text\":\"        params.append(entity_type)\"},{\"line\":338,\"text\":\"    if lifecycle_status:\"},{\"line\":339,\"text\":\"        clauses.append(\\\"e.lifecycle_status = %s\\\")\"},{\"line\":340,\"text\":\"        params.append(lifecycle_status)\"},{\"line\":341,\"text\":\"    if source_of_truth:\"},{\"line\":342,\"text\":\"        clauses.append(\\\"e.source_of_truth = %s\\\")\"},{\"line\":343,\"text\":\"        params.append(source_of_truth)\"},{\"line\":344,\"text\":\"    since = api_v1_updated_since(updated_since)\"},{\"line\":345,\"text\":\"    if since is not None:\"},{\"line\":346,\"text\":\"        clauses.append(\\\"e.updated_at >= %s\\\")\"},{\"line\":347,\"text\":\"        params.append(since)\"},{\"line\":348,\"text\":\"    if search:\"},{\"line\":349,\"text\":\"        clauses.append(\\\"(e.entity_key ILIKE %s OR e.display_name ILIKE %s)\\\")\"},{\"line\":350,\"text\":\"        params.extend([\\\"%\\\" + search + \\\"%\\\", \\\"%\\\" + search + \\\"%\\\"])\"},{\"line\":351,\"text\":\"    sql = \\\"SELECT e.entity_key,e.entity_type,e.display_name,e.lifecycle_status,e.source_of_truth,e.created_at,e.updated_at FROM entities e\\\"\"},{\"line\":352,\"text\":\"    if clauses:\"},{\"line\":353,\"text\":\"        sql += \\\" WHERE \\\" + \\\" AND \\\".join(clauses)\"},{\"line\":354,\"text\":\"    sql += \\\" ORDER BY e.entity_key LIMIT %s\\\"\"},{\"line\":355,\"text\":\"    params.append(limit + 1)\"},{\"line\":356,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":357,\"text\":\"        db_cursor.execute(sql, params)\"},{\"line\":358,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":359,\"text\":\"    page = rows[:limit]\"},{\"line\":360,\"text\":\"    next_cursor = page[-1][\\\"entity_key\\\"] if len(rows) > limit else None\"},{\"line\":361,\"text\":\"    return {\\\"items\\\": page, \\\"next_cursor\\\": next_cursor, \\\"limit\\\": limit}\"},{\"line\":362,\"text\":\"\"},{\"line\":363,\"text\":\"\"},{\"line\":364,\"text\":\"@app.get(\\\"/api/v1/entities/{entity_key:path}/dependencies\\\")\"},{\"line\":365,\"text\":\"def api_v1_entity_dependencies(request: Request, entity_key: str, limit: int = 100, cursor: str = \\\"\\\"):\"},{\"line\":366,\"text\":\"    api_v1_require_session(request)\"},{\"line\":367,\"text\":\"    limit = api_v1_limit(limit)\"},{\"line\":368,\"text\":\"    cursor_id = api_v1_decode_id_cursor(cursor)\"},{\"line\":369,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":370,\"text\":\"        db_cursor.execute(\\\"SELECT id FROM entities WHERE entity_key=%s\\\", (entity_key,))\"},{\"line\":371,\"text\":\"        entity = db_cursor.fetchone()\"},{\"line\":372,\"text\":\"        if not entity:\"},{\"line\":373,\"text\":\"            raise HTTPException(404, detail=\\\"entity not found\\\")\"},{\"line\":374,\"text\":\"        entity_id = entity[\\\"id\\\"]\"},{\"line\":375,\"text\":\"        clauses = [\\\"(d.parent_entity_id=%s OR d.child_entity_id=%s)\\\"]\"},{\"line\":376,\"text\":\"        params = [entity_id, entity_id, entity_id]\"},{\"line\":377,\"text\":\"        if cursor_id is not None:\"},{\"line\":378,\"text\":\"            clauses.append(\\\"d.id > %s\\\")\"},{\"line\":379,\"text\":\"            params.append(cursor_id)\"},{\"line\":380,\"text\":\"        sql = \\\"SELECT d.id AS _cursor_id,CASE WHEN d.parent_entity_id=%s THEN 'outgoing' ELSE 'incoming' END AS direction,p.entity_key AS parent_entity_key,c.entity_key AS child_entity_key,d.relation_type,d.source_relation,d.criticality,d.edge_state,d.created_at,d.updated_at FROM entity_dependencies d JOIN entities p ON p.id=d.parent_entity_id JOIN entities c ON c.id=d.child_entity_id WHERE \\\" + \\\" AND \\\".join(clauses) + \\\" ORDER BY d.id LIMIT %s\\\"\"},{\"line\":381,\"text\":\"        params.append(limit + 1)\"},{\"line\":382,\"text\":\"        db_cursor.execute(sql, params)\"},{\"line\":383,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":384,\"text\":\"    page = rows[:limit]\"},{\"line\":385,\"text\":\"    next_cursor = api_v1_encode_id_cursor(page[-1][\\\"_cursor_id\\\"]) if len(rows) > limit else None\"},{\"line\":386,\"text\":\"    for row in page:\"},{\"line\":387,\"text\":\"        row.pop(\\\"_cursor_id\\\", None)\"},{\"line\":388,\"text\":\"    return {\\\"entity_key\\\": entity_key, \\\"items\\\": page, \\\"next_cursor\\\": next_cursor, \\\"limit\\\": limit}\"},{\"line\":389,\"text\":\"\"},{\"line\":390,\"text\":\"\"},{\"line\":391,\"text\":\"@app.get(\\\"/api/v1/entities/{entity_key:path}\\\")\"},{\"line\":392,\"text\":\"def api_v1_entity_detail(request: Request, entity_key: str):\"},{\"line\":393,\"text\":\"    api_v1_require_session(request)\"},{\"line\":394,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":395,\"text\":\"        db_cursor.execute(\\\"SELECT id AS _entity_id,entity_key,entity_type,display_name,lifecycle_status,source_of_truth,legacy_table,created_at,updated_at FROM entities WHERE entity_key=%s\\\", (entity_key,))\"},{\"line\":396,\"text\":\"        row = db_cursor.fetchone()\"},{\"line\":397,\"text\":\"        if not row:\"},{\"line\":398,\"text\":\"            raise HTTPException(404, detail=\\\"entity not found\\\")\"},{\"line\":399,\"text\":\"        result = dict(row)\"},{\"line\":400,\"text\":\"        entity_id = result.pop(\\\"_entity_id\\\")\"},{\"line\":401,\"text\":\"        db_cursor.execute(\\\"SELECT alias,alias_type,priority,is_canonical,created_at FROM entity_aliases WHERE entity_id=%s ORDER BY is_canonical DESC,priority,alias,alias_type\\\", (entity_id,))\"},{\"line\":402,\"text\":\"        result[\\\"aliases\\\"] = [dict(alias) for alias in db_cursor.fetchall()]\"},{\"line\":403,\"text\":\"    return result\"},{\"line\":404,\"text\":\"\"},{\"line\":405,\"text\":\"\"},{\"line\":406,\"text\":\"@app.get(\\\"/api/v1/dependencies\\\")\"},{\"line\":407,\"text\":\"def api_v1_dependencies(request: Request, limit: int = 100, cursor: str = \\\"\\\", parent_entity_key: str = \\\"\\\", child_entity_key: str = \\\"\\\", relation_type: str = \\\"\\\", source_relation: str = \\\"\\\", criticality: str = \\\"\\\", edge_state: str = \\\"\\\"):\"},{\"line\":408,\"text\":\"    api_v1_require_session(request)\"},{\"line\":409,\"text\":\"    limit = api_v1_limit(limit)\"},{\"line\":410,\"text\":\"    cursor_id = api_v1_decode_id_cursor(cursor)\"},{\"line\":411,\"text\":\"    clauses = []\"},{\"line\":412,\"text\":\"    params = []\"},{\"line\":413,\"text\":\"    if cursor_id is not None:\"},{\"line\":414,\"text\":\"        clauses.append(\\\"d.id > %s\\\")\"},{\"line\":415,\"text\":\"        params.append(cursor_id)\"},{\"line\":416,\"text\":\"    filters = [(\\\"p.entity_key\\\", parent_entity_key), (\\\"c.entity_key\\\", child_entity_key), (\\\"d.relation_type\\\", relation_type), (\\\"d.source_relation\\\", source_relation), (\\\"d.criticality\\\", criticality), (\\\"d.edge_state\\\", edge_state)]\"},{\"line\":417,\"text\":\"    for column, value in filters:\"},{\"line\":418,\"text\":\"        if value:\"},{\"line\":419,\"text\":\"            clauses.append(column + \\\" = %s\\\")\"},{\"line\":420,\"text\":\"            params.append(value)\"},{\"line\":421,\"text\":\"    sql = \\\"SELECT d.id AS _cursor_id,p.entity_key AS parent_entity_key,c.entity_key AS child_entity_key,p.entity_type AS parent_entity_type,c.entity_type AS child_entity_type,d.relation_type,d.source_relation,d.criticality,d.edge_state,d.created_at,d.updated_at FROM entity_dependencies d JOIN entities p ON p.id=d.parent_entity_id JOIN entities c ON c.id=d.child_entity_id\\\"\"},{\"line\":422,\"text\":\"    if clauses:\"},{\"line\":423,\"text\":\"        sql += \\\" WHERE \\\" + \\\" AND \\\".join(clauses)\"},{\"line\":424,\"text\":\"    sql += \\\" ORDER BY d.id LIMIT %s\\\"\"},{\"line\":425,\"text\":\"    params.append(limit + 1)\"},{\"line\":426,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":427,\"text\":\"        db_cursor.execute(sql, params)\"},{\"line\":428,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":429,\"text\":\"    page = rows[:limit]\"},{\"line\":430,\"text\":\"    next_cursor = api_v1_encode_id_cursor(page[-1][\\\"_cursor_id\\\"]) if len(rows) > limit else None\"},{\"line\":431,\"text\":\"    for row in page:\"},{\"line\":432,\"text\":\"        row.pop(\\\"_cursor_id\\\", None)\"},{\"line\":433,\"text\":\"    return {\\\"items\\\": page, \\\"next_cursor\\\": next_cursor, \\\"limit\\\": limit}\"},{\"line\":434,\"text\":\"\"},{\"line\":435,\"text\":\"\"},{\"line\":436,\"text\":\"@app.get(\\\"/api/v1/sources\\\")\"},{\"line\":437,\"text\":\"def api_v1_sources(request: Request):\"},{\"line\":438,\"text\":\"    api_v1_require_session(request)\"},{\"line\":439,\"text\":\"    sql = \\\"SELECT s.source_key,s.source_type,s.enabled,si.instance_key,si.status AS instance_status,si.last_seen,se.expectation_type,se.interval_seconds,se.grace_seconds,se.severity,s.created_at,s.updated_at FROM sources s LEFT JOIN source_instances si ON si.source_id=s.id LEFT JOIN source_expectations se ON se.source_instance_id=si.id ORDER BY s.source_key,si.instance_key,se.expectation_type\\\"\"},{\"line\":440,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":441,\"text\":\"        db_cursor.execute(sql)\"},{\"line\":442,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":443,\"text\":\"    return {\\\"items\\\": rows}\"},{\"line\":444,\"text\":\"\"},{\"line\":445,\"text\":\"\"},{\"line\":446,\"text\":\"@app.get(\\\"/api/v1/policies\\\")\"},{\"line\":447,\"text\":\"def api_v1_policies(request: Request):\"},{\"line\":448,\"text\":\"    api_v1_require_session(request)\"},{\"line\":449,\"text\":\"    sql = \\\"SELECT pr.policy_key,pr.scope,pr.enabled,pr.active_version,pv.document->>'mode' AS mode,pv.document->>'automatic_actions' AS automatic_actions,pv.document->>'human_approval_required' AS human_approval_required,pr.created_at,pr.updated_at FROM policy_registry pr LEFT JOIN policy_versions pv ON pv.policy_id=pr.id AND pv.version=pr.active_version ORDER BY pr.policy_key,pr.scope\\\"\"},{\"line\":450,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":451,\"text\":\"        db_cursor.execute(sql)\"},{\"line\":452,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":453,\"text\":\"    return {\\\"items\\\": rows}\"},{\"line\":454,\"text\":\"\"},{\"line\":455,\"text\":\"\"},{\"line\":456,\"text\":\"@app.get(\\\"/api/v1/collector-runs\\\")\"},{\"line\":457,\"text\":\"def api_v1_collector_runs(request: Request, limit: int = 100, cursor: str = \\\"\\\", collector: str = \\\"\\\", status: str = \\\"\\\"):\"},{\"line\":458,\"text\":\"    api_v1_require_session(request)\"},{\"line\":459,\"text\":\"    limit = api_v1_limit(limit)\"},{\"line\":460,\"text\":\"    cursor_id = api_v1_decode_id_cursor(cursor)\"},{\"line\":461,\"text\":\"    clauses = []\"},{\"line\":462,\"text\":\"    params = []\"},{\"line\":463,\"text\":\"    if cursor_id is not None:\"},{\"line\":464,\"text\":\"        clauses.append(\\\"cr.id < %s\\\")\"},{\"line\":465,\"text\":\"        params.append(cursor_id)\"},{\"line\":466,\"text\":\"    if collector:\"},{\"line\":467,\"text\":\"        clauses.append(\\\"cr.collector = %s\\\")\"},{\"line\":468,\"text\":\"        params.append(collector)\"},{\"line\":469,\"text\":\"    if status:\"},{\"line\":470,\"text\":\"        clauses.append(\\\"cr.status = %s\\\")\"},{\"line\":471,\"text\":\"        params.append(status)\"},{\"line\":472,\"text\":\"    sql = \\\"SELECT cr.id AS _cursor_id,cr.collector,cr.status,cr.trigger_kind,cr.schema_version,cr.records_seen,cr.warning_count,cr.run_key,si.instance_key AS source_instance_key,cr.started_at,cr.finished_at FROM collector_runs cr LEFT JOIN source_instances si ON si.id=cr.source_instance_id\\\"\"},{\"line\":473,\"text\":\"    if clauses:\"},{\"line\":474,\"text\":\"        sql += \\\" WHERE \\\" + \\\" AND \\\".join(clauses)\"},{\"line\":475,\"text\":\"    sql += \\\" ORDER BY cr.id DESC LIMIT %s\\\"\"},{\"line\":476,\"text\":\"    params.append(limit + 1)\"},{\"line\":477,\"text\":\"    with api_v1_read_cursor() as db_cursor:\"},{\"line\":478,\"text\":\"        db_cursor.execute(sql, params)\"},{\"line\":479,\"text\":\"        rows = [dict(row) for row in db_cursor.fetchall()]\"},{\"line\":480,\"text\":\"    page = rows[:limit]\"},{\"line\":481,\"text\":\"    next_cursor = api_v1_encode_id_cursor(page[-1][\\\"_cursor_id\\\"]) if len(rows) > limit else None\"},{\"line\":482,\"text\":\"    for row in page:\"},{\"line\":483,\"text\":\"        row.pop(\\\"_cursor_id\\\", None)\"},{\"line\":484,\"text\":\"    return {\\\"items\\\": page, \\\"next_cursor\\\": next_cursor, \\\"limit\\\": limit}\"}]},{\"bytes\":10640,\"line_count\":113,\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"priority\":50,\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\",\"source\":[{\"line\":1,\"text\":\"#!/usr/bin/env python3\"},{\"line\":2,\"text\":\"import datetime\"},{\"line\":3,\"text\":\"import json\"},{\"line\":4,\"text\":\"import re\"},{\"line\":5,\"text\":\"import urllib.parse\"},{\"line\":6,\"text\":\"import urllib.request\"},{\"line\":7,\"text\":\"import psycopg2\"},{\"line\":8,\"text\":\"from psycopg2.extras import Json\"},{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":11,\"text\":\"PROMETHEUS = \\\"http://[PRIVATE_IP]:9090\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":13,\"text\":\"def prometheus(expression):\"},{\"line\":14,\"text\":\"    try:\"},{\"line\":15,\"text\":\"        url = PROMETHEUS + \\\"/api/v1/query?\\\" + urllib.parse.urlencode({\\\"query\\\":expression})\"},{\"line\":16,\"text\":\"        with urllib.request.urlopen(url, timeout=12) as response:\"},{\"line\":17,\"text\":\"            body = json.loads(response.read().decode(\\\"utf-8\\\"))\"},{\"line\":18,\"text\":\"        return body.get(\\\"data\\\", {}).get(\\\"result\\\", []) if body.get(\\\"status\\\") == \\\"success\\\" else []\"},{\"line\":19,\"text\":\"    except Exception:\"},{\"line\":20,\"text\":\"        return []\"},{\"line\":21,\"text\":\"def numeric(value):\"},{\"line\":22,\"text\":\"    try:\"},{\"line\":23,\"text\":\"        return float(value)\"},{\"line\":24,\"text\":\"    except Exception:\"},{\"line\":25,\"text\":\"        return None\"},{\"line\":26,\"text\":\"def slug(value):\"},{\"line\":27,\"text\":\"    return re.sub(r\\\"[^a-z0-9]+\\\", \\\"-\\\", str(value).lower()).strip(\\\"-\\\")\"},{\"line\":28,\"text\":\"def entity(instance):\"},{\"line\":29,\"text\":\"    address = instance.rsplit(\\\":\\\", 1)[0]\"},{\"line\":30,\"text\":\"    return HOSTS.get(address, address)\"},{\"line\":31,\"text\":\"def open_incident(cursor, fingerprint, severity, title, summary, cause, metadata):\"},{\"line\":32,\"text\":\"    cursor.execute(\\\"INSERT INTO incidents(fingerprint,status,severity,title,summary,root_cause,first_seen,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,now(),now(),%s) ON CONFLICT(fingerprint) DO UPDATE SET status=EXCLUDED.status,severity=EXCLUDED.severity,title=EXCLUDED.title,summary=EXCLUDED.summary,root_cause=EXCLUDED.root_cause,last_seen=now(),recovered_at=NULL,metadata=EXCLUDED.metadata RETURNING id\\\",(fingerprint,\\\"OPEN\\\",severity,title,summary,cause,Json(metadata)))\"},{\"line\":33,\"text\":\"    incident_id = cursor.fetchone()[0]\"},{\"line\":34,\"text\":\"    cursor.execute(\\\"INSERT INTO incident_events(incident_id,event_type,message,metadata) SELECT %s,%s,%s,%s WHERE NOT EXISTS(SELECT 1 FROM incident_events WHERE incident_id=%s AND event_type=%s AND created_at>now()-make_interval(mins=>10))\\\",(incident_id,\\\"OBSERVED\\\",summary,Json(metadata),incident_id,\\\"OBSERVED\\\"))\"},{\"line\":35,\"text\":\"def main():\"},{\"line\":36,\"text\":\"    data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\"    connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":38,\"text\":\"    active = set()\"},{\"line\":39,\"text\":\"    metric_count = 0\"},{\"line\":40,\"text\":\"    with connection.cursor() as cursor:\"},{\"line\":41,\"text\":\"        cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":42,\"text\":\"        run_id = cursor.fetchone()[0]\"},{\"line\":43,\"text\":\"        for item in data.get(\\\"nodes\\\", []):\"},{\"line\":44,\"text\":\"            cursor.execute(\\\"INSERT INTO nodes(name,address,role,status,last_seen,metadata) VALUES(%s,%s,%s,%s,now(),%s) ON CONFLICT(name) DO UPDATE SET address=EXCLUDED.address,role=EXCLUDED.role,status=EXCLUDED.status,last_seen=now(),metadata=EXCLUDED.metadata\\\",(item.get(\\\"name\\\"),item.get(\\\"address\\\"),item.get(\\\"role\\\"),item.get(\\\"status\\\"),Json(item)))\"},{\"line\":45,\"text\":\"        for item in data.get(\\\"guests\\\", []):\"},{\"line\":46,\"text\":\"            cursor.execute(\\\"INSERT INTO guests(guest_key,node_name,guest_type,name,status,cpu_usage,memory_used_bytes,memory_total_bytes,disk_used_bytes,disk_total_bytes,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,now(),%s) ON CONFLICT(guest_key) DO UPDATE SET node_name=EXCLUDED.node_name,guest_type=EXCLUDED.guest_type,name=EXCLUDED.name,status=EXCLUDED.status,cpu_usage=EXCLUDED.cpu_usage,memory_used_bytes=EXCLUDED.memory_used_bytes,memory_total_bytes=EXCLUDED.memory_total_bytes,disk_used_bytes=EXCLUDED.disk_used_bytes,disk_total_bytes=EXCLUDED.disk_total_bytes,last_seen=now(),metadata=EXCLUDED.metadata\\\",(item.get(\\\"guest_key\\\"),item.get(\\\"node_name\\\"),item.get(\\\"guest_type\\\"),item.get(\\\"name\\\"),item.get(\\\"status\\\"),item.get(\\\"cpu_usage\\\"),item.get(\\\"memory_used_bytes\\\"),item.get(\\\"memory_total_bytes\\\"),item.get(\\\"disk_used_bytes\\\"),item.get(\\\"disk_total_bytes\\\"),Json(item)))\"},{\"line\":47,\"text\":\"        for item in data.get(\\\"services\\\", []):\"},{\"line\":48,\"text\":\"            service_key = item.get(\\\"service_key\\\") or slug(item.get(\\\"name\\\"))\"},{\"line\":49,\"text\":\"            cursor.execute(\\\"INSERT INTO services(service_key,name,scope,host_name,container_name,public_url,internal_url,status,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,now(),%s) ON CONFLICT(service_key) DO UPDATE SET name=EXCLUDED.name,scope=EXCLUDED.scope,host_name=EXCLUDED.host_name,container_name=EXCLUDED.container_name,public_url=EXCLUDED.public_url,internal_url=EXCLUDED.internal_url,status=EXCLUDED.status,last_seen=now(),metadata=EXCLUDED.metadata\\\",(service_key,item.get(\\\"name\\\"),item.get(\\\"scope\\\"),item.get(\\\"host_name\\\"),item.get(\\\"container_name\\\"),item.get(\\\"public_url\\\"),item.get(\\\"internal_url\\\"),item.get(\\\"status\\\"),Json(item)))\"},{\"line\":50,\"text\":\"            if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":51,\"text\":\"                fingerprint = \\\"service:\\\" + service_key\"},{\"line\":52,\"text\":\"                active.add(fingerprint)\"},{\"line\":53,\"text\":\"                open_incident(cursor,fingerprint,\\\"P1\\\",\\\"Service requires attention: \\\" + str(item.get(\\\"name\\\")),\\\"Readiness status is \\\" + str(item.get(\\\"status\\\")),\\\"service-readiness\\\",item)\"},{\"line\":54,\"text\":\"        cursor.execute(\\\"DELETE FROM dependencies\\\")\"},{\"line\":55,\"text\":\"        for item in data.get(\\\"dependencies\\\", []):\"},{\"line\":56,\"text\":\"            cursor.execute(\\\"INSERT INTO dependencies(parent_key,child_key,relation,metadata) VALUES(%s,%s,%s,%s) ON CONFLICT(parent_key,child_key,relation) DO UPDATE SET metadata=EXCLUDED.metadata\\\",(item.get(\\\"parent_key\\\"),item.get(\\\"child_key\\\"),item.get(\\\"relation\\\"),Json(item)))\"},{\"line\":57,\"text\":\"        cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":58,\"text\":\"        expressions = {\\\"up\\\":\\\"up\\\",\\\"cpu_used_pct\\\":\\\"100-(avg by(instance)(rate(node_cpu_seconds_total{mode=\\\\\\\"idle\\\\\\\"}[5m]))*100)\\\",\\\"memory_used_pct\\\":\\\"100*(1-(node_memory_MemAvailable_bytes/node_memory_MemTotal_bytes))\\\",\\\"filesystem_used_pct\\\":\\\"100*(1-(node_filesystem_avail_bytes{fstype!~\\\\\\\"tmpfs|overlay|squashfs|nsfs\\\\\\\"}/node_filesystem_size_bytes{fstype!~\\\\\\\"tmpfs|overlay|squashfs|nsfs\\\\\\\"}))\\\",\\\"load1\\\":\\\"node_load1\\\"}\"},{\"line\":59,\"text\":\"        for metric, expression in expressions.items():\"},{\"line\":60,\"text\":\"            for result in prometheus(expression):\"},{\"line\":61,\"text\":\"                labels = result.get(\\\"metric\\\", {})\"},{\"line\":62,\"text\":\"                instance = labels.get(\\\"instance\\\", \\\"unknown\\\")\"},{\"line\":63,\"text\":\"                entity_key = entity(instance)\"},{\"line\":64,\"text\":\"                value = numeric(result.get(\\\"value\\\", [None, None])[1])\"},{\"line\":65,\"text\":\"                if value is None:\"},{\"line\":66,\"text\":\"                    continue\"},{\"line\":67,\"text\":\"                status = \\\"OK\\\"\"},{\"line\":68,\"text\":\"                severity = \\\"P2\\\"\"},{\"line\":69,\"text\":\"                if metric == \\\"up\\\" and value < 1:\"},{\"line\":70,\"text\":\"                    status, severity = \\\"BAD\\\", \\\"P1\\\"\"},{\"line\":71,\"text\":\"                elif metric == \\\"cpu_used_pct\\\" and value >= 97:\"},{\"line\":72,\"text\":\"                    status, severity = \\\"BAD\\\", \\\"P1\\\"\"},{\"line\":73,\"text\":\"                elif metric == \\\"cpu_used_pct\\\" and value >= 90:\"},{\"line\":74,\"text\":\"                    status = \\\"WARN\\\"\"},{\"line\":75,\"text\":\"                elif metric == \\\"memory_used_pct\\\" and value >= 95:\"},{\"line\":76,\"text\":\"                    status, severity = \\\"BAD\\\", \\\"P1\\\"\"},{\"line\":77,\"text\":\"                elif metric == \\\"memory_used_pct\\\" and value >= 88:\"},{\"line\":78,\"text\":\"                    status = \\\"WARN\\\"\"},{\"line\":79,\"text\":\"                elif metric == \\\"filesystem_used_pct\\\" and value >= 95:\"},{\"line\":80,\"text\":\"                    status, severity = \\\"BAD\\\", \\\"P1\\\"\"},{\"line\":81,\"text\":\"                elif metric == \\\"filesystem_used_pct\\\" and value >= 85:\"},{\"line\":82,\"text\":\"                    status = \\\"WARN\\\"\"},{\"line\":83,\"text\":\"                unit = \\\"percent\\\" if metric.endswith(\\\"pct\\\") else \\\"state\\\" if metric == \\\"up\\\" else \\\"load\\\"\"},{\"line\":84,\"text\":\"                metadata = {\\\"instance\\\":instance,\\\"job\\\":labels.get(\\\"job\\\"),\\\"mountpoint\\\":labels.get(\\\"mountpoint\\\"),\\\"device\\\":labels.get(\\\"device\\\")}\"},{\"line\":85,\"text\":\"                cursor.execute(\\\"INSERT INTO observations(entity_type,entity_key,metric,value,unit,status,observed_at,metadata) VALUES(%s,%s,%s,%s,%s,%s,now(),%s)\\\",(\\\"target\\\",entity_key,metric,value,unit,status,Json(metadata)))\"},{\"line\":86,\"text\":\"                metric_count += 1\"},{\"line\":87,\"text\":\"                if status != \\\"OK\\\":\"},{\"line\":88,\"text\":\"                    cursor.execute(\\\"SELECT count(*) FROM observations WHERE entity_type=%s AND entity_key=%s AND metric=%s AND status=ANY(%s) AND observed_at>now()-make_interval(mins=>10)\\\",(\\\"target\\\",entity_key,metric,[\\\"WARN\\\",\\\"BAD\\\"]))\"},{\"line\":89,\"text\":\"                    if cursor.fetchone()[0] >= 2:\"},{\"line\":90,\"text\":\"                        fingerprint = \\\"metric:\\\" + slug(entity_key) + \\\":\\\" + metric + \\\":\\\" + slug(labels.get(\\\"mountpoint\\\", \\\"root\\\"))\"},{\"line\":91,\"text\":\"                        active.add(fingerprint)\"},{\"line\":92,\"text\":\"                        open_incident(cursor,fingerprint,severity,entity_key + \\\" - \\\" + metric,metric + \\\"=\\\" + format(value, \\\".2f\\\") + \\\" \\\" + unit,\\\"target-unavailable\\\" if metric == \\\"up\\\" else \\\"resource-pressure\\\",metadata)\"},{\"line\":93,\"text\":\"        for name, state in data.get(\\\"backups\\\", {}).items():\"},{\"line\":94,\"text\":\"            if state.get(\\\"status\\\") != \\\"success\\\" or not state.get(\\\"local_backup_deleted\\\", False) or not state.get(\\\"remote_a_job_path\\\") or not state.get(\\\"remote_b_job_path\\\"):\"},{\"line\":95,\"text\":\"                fingerprint = \\\"backup:\\\" + slug(name)\"},{\"line\":96,\"text\":\"                active.add(fingerprint)\"},{\"line\":97,\"text\":\"                open_incident(cursor,fingerprint,\\\"P1\\\",\\\"Backup requires attention: \\\" + name,\\\"Backup is not fully verified\\\",\\\"backup-verification\\\",state)\"},{\"line\":98,\"text\":\"        cursor.execute(\\\"SELECT id,fingerprint FROM incidents WHERE status=ANY(%s)\\\",([\\\"OPEN\\\",\\\"DETECTED\\\",\\\"ACKNOWLEDGED\\\",\\\"MONITORING\\\"],))\"},{\"line\":99,\"text\":\"        for incident_id, fingerprint in cursor.fetchall():\"},{\"line\":100,\"text\":\"            if fingerprint not in active:\"},{\"line\":101,\"text\":\"                cursor.execute(\\\"UPDATE incidents SET status=%s,recovered_at=now(),last_seen=now() WHERE id=%s\\\",(\\\"RECOVERED\\\",incident_id))\"},{\"line\":102,\"text\":\"                cursor.execute(\\\"INSERT INTO incident_events(incident_id,event_type,message,metadata) VALUES(%s,%s,%s,%s)\\\",(incident_id,\\\"RECOVERED\\\",\\\"Condition returned to normal\\\",Json({})))\"},{\"line\":103,\"text\":\"        cursor.execute(\\\"DELETE FROM observations WHERE observed_at<now()-make_interval(days=>30)\\\")\"},{\"line\":104,\"text\":\"        cursor.execute(\\\"UPDATE collector_runs SET status=%s,finished_at=now(),metadata=%s WHERE id=%s\\\",(\\\"OK\\\",Json({\\\"nodes\\\":len(data.get(\\\"nodes\\\",[])),\\\"guests\\\":len(data.get(\\\"guests\\\",[])),\\\"services\\\":len(data.get(\\\"services\\\",[])),\\\"metrics\\\":metric_count}),run_id))\"},{\"line\":105,\"text\":\"        cursor.execute(\\\"SELECT count(*) FROM incidents WHERE status=ANY(%s)\\\",([\\\"OPEN\\\",\\\"DETECTED\\\",\\\"ACKNOWLEDGED\\\",\\\"MONITORING\\\"],))\"},{\"line\":106,\"text\":\"        open_count = cursor.fetchone()[0]\"},{\"line\":107,\"text\":\"    connection.commit()\"},{\"line\":108,\"text\":\"    line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"},{\"line\":109,\"text\":\"    with open(HEALTH, \\\"w\\\", encoding=\\\"utf-8\\\") as stream:\"},{\"line\":110,\"text\":\"        stream.write(line)\"},{\"line\":111,\"text\":\"    print(line.strip())\"},{\"line\":112,\"text\":\"if __name__ == \\\"__main__\\\":\"},{\"line\":113,\"text\":\"    main()\"}]}],\"gate\":{\"exact_source_count\":30,\"index_source_count\":1,\"ready_for_atomic_design\":true,\"reference_source_count\":1,\"unit_record_count\":16},\"hostname\":\"cluster-admin\",\"index_paths\":[\"/var/www/homelab-cluster-admin/index.html\"],\"input_counts\":{\"application_candidates\":40,\"health_input_files\":4,\"indexes\":1,\"reference_files\":1,\"units\":16},\"reference_paths\":[\"/opt/homelab-cluster-admin/render-webpanel.sh\"],\"unit_records\":[{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006178 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:27 UTC] ; stop_time=[Wed 2026-08-05 13:20:28 UTC] ; pid=2006164 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:15 UTC] ; pid=2005776 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005777 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005778 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006179 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:11:27 UTC] ; stop_time=[Wed 2026-08-05 13:11:27 UTC] ; pid=2004608 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"Group\":\"root\",\"LoadState\":\"loaded\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"root\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"cat_stderr\":\"\",\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}]},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":18,\"status\":\"OK\",\"transaction_gate\":{\"index_source_found\":true,\"next_required_action\":\"build corrected atomic local+remote transaction\",\"ready_for_atomic_transaction\":true,\"reference_source_found\":true,\"remote_probe_root\":true,\"unit_records_found\":true},\"transport\":{\"attempts\":[{\"privilege\":\"sudo-root\",\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":139487,\"stdout_sha256\":\"a710627984d38927f5e0fdde30c8beab966ad52581868b9928ea458dc6902bf3\"}],\"privilege\":\"sudo-root\",\"success\":true},\"v15_candidate_manifest\":{\"application_candidates\":[{\"bytes\":133,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},{\"bytes\":149,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},{\"bytes\":151,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},{\"bytes\":231,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},{\"bytes\":138,\"matches\":[{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},{\"bytes\":127,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},{\"bytes\":616,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},{\"bytes\":1190,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},{\"bytes\":223,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},{\"bytes\":803,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},{\"bytes\":131,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},{\"bytes\":301,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},{\"bytes\":89,\"matches\":[{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\"},{\"bytes\":8444,\"matches\":[{\"line\":4,\"text\":\"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')\"},{\"line\":64,\"text\":\" return f\\\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\\\n\\\"\"},{\"line\":82,\"text\":\" old=<REDACTED>\"},{\"line\":97,\"text\":\"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)\"},{\"line\":99,\"text\":\"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\"},{\"bytes\":28599,\"matches\":[{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =<REDACTED>\"},{\"line\":14,\"text\":\"COOKIE =<REDACTED>\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":20,\"text\":\"    return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":73,\"text\":\"    return f\\\"\\\"\\\"<!doctype html><html lang=<REDACTED>\"},{\"line\":152,\"text\":\"    return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":220,\"text\":\"    page = f\\\"\\\"\\\"<!doctype html><html lang=\\\"ru\\\"><head><meta charset=\\\"utf-8\\\"><meta name=\\\"viewport\\\" content=\\\"width=device-width,initial-scale=1\\\"><meta http-equiv=\\\"refresh\\\" content=\\\"30\\\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\\\"post\\\" action=\\\"/logout\\\"><input type=\\\"hidden\\\" name=\\\"csrf\\\" value=\\\"{csrf}\\\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\\\"\\\"\\\"\"},{\"line\":243,\"text\":\"        \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\"        \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\"        \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\"        \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\"        \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\"        \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\"        \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\"        \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\"        \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\"        \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\"        \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\"        \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\"        \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\"        \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\"        \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\"        \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\"        \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\"        \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\"        \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\"        \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\"        \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":267,\"text\":\"        lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\"},{\"bytes\":922,\"matches\":[{\"line\":2,\"text\":\"first = getpass.getpass(\\\"Новый пароль Cluster Admin: \\\")\"},{\"line\":12,\"text\":\"directory = \\\"/etc/cluster-admin-incident-engine\\\"\"},{\"line\":16,\"text\":\"account = pwd.getpwnam(\\\"clusteradmin\\\")\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\"},{\"bytes\":10640,\"matches\":[{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":36,\"text\":\"    data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\"    connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":41,\"text\":\"        cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":50,\"text\":\"            if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":57,\"text\":\"        cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":108,\"text\":\"    line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\"},{\"bytes\":1857,\"matches\":[{\"line\":3,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage1-foundation-migration-plan\\\",\"},{\"line\":43,\"text\":\"    \\\"function\\\": \\\"cluster_admin_enrich_collector_run\\\",\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\"},{\"bytes\":1342,\"matches\":[{\"line\":3,\"text\":\"  \\\"collector_call\\\": \\\"SELECT cluster_admin_sync_foundation(%s)\\\",\"},{\"line\":15,\"text\":\"  \\\"function\\\": \\\"cluster_admin_sync_foundation\\\",\"},{\"line\":16,\"text\":\"  \\\"function_signature\\\": \\\"cluster_admin_sync_foundation(bigint)\\\",\"},{\"line\":26,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-migration-plan\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\"},{\"bytes\":2355,\"matches\":[{\"line\":51,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-dynamic-acceptance-contract\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\"},{\"bytes\":2832,\"matches\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\"  line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\"  echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},{\"bytes\":3471,\"matches\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\"  echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\"  echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},{\"bytes\":1433,\"matches\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},{\"bytes\":848,\"matches\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":10353,\"matches\":[{\"line\":7,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":19,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":24,\"text\":\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\"},{\"line\":28,\"text\":\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"},{\"line\":29,\"text\":\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\"},{\"line\":32,\"text\":\"<div class=<REDACTED>\"},{\"line\":33,\"text\":\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\"},{\"line\":34,\"text\":\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\"},{\"line\":35,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":46,\"text\":\"OK severity=<REDACTED>\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\"},{\"bytes\":168,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729\"},{\"bytes\":259,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741\"},{\"bytes\":134,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517\"},{\"bytes\":141,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\"},{\"bytes\":189,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\"},{\"bytes\":131,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":199,\"matches\":[{\"line\":1,\"text\":\"STATUS=<REDACTED>\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\"},{\"bytes\":272,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec\"},{\"bytes\":380,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"}],\"gate\":{\"application_candidate_count\":40,\"ready_for_remote_transaction_design\":true,\"reference_file_count\":1,\"unit_count\":16},\"health_input_files\":[{\"bytes\":380,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"},{\"bytes\":128,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\"},{\"bytes\":1535,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\"}],\"indexes\":[{\"bytes\":10353,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"root\":\"/var/www/homelab-cluster-admin\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\",\"title_lines\":[\"<title>Homelab Cluster Admin</title>\",\"<h1>Homelab Cluster Admin</h1>\",\"<div class=\\\"card\\\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"<div class=\\\"card\\\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>\",\"<div class=<REDACTED>\",\"<div class=\\\"card\\\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>\",\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer\",\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"]}],\"listeners\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"State  Recv-Q Send-Q Local Address:Port Peer Address:PortProcess                                   \\nLISTEN 0      5            0.0.0.0:8080      0.0.0.0:*    users:((\\\"python3\\\",pid=439,fd=3))         \\nLISTEN 0      2048         0.0.0.0:8081      0.0.0.0:*    users:((\\\"python3\\\",pid=487,fd=6))         \\nLISTEN 0      244        127.0.0.1:5432      0.0.0.0:*    users:((\\\"postgres\\\",pid=469,fd=6))        \\nLISTEN 0      4096   127.0.0.53%lo:53        0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=18))\\nLISTEN 0      4096         0.0.0.0:5355      0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=12))\\nLISTEN 0      4096      127.0.0.54:53        0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=20))\\nLISTEN 0      128          0.0.0.0:22        0.0.0.0:*    users:((\\\"sshd\\\",pid=457,fd=3))            \\nLISTEN 0      244            [::1]:5432         [::]:*    users:((\\\"postgres\\\",pid=469,fd=5))        \\nLISTEN 0      4096            [::]:5355         [::]:*    users:((\\\"systemd-resolve\\\",pid=380,fd=14))\\nLISTEN 0      128             [::]:22           [::]:*    users:((\\\"sshd\\\",pid=457,fd=4))            \"},\"process_lines\":[\"    439       1 root     root     python3         /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\",\"    487       1 cluster+ cluster+ python3         /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\"],\"reference_files\":[{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"<title>Homelab Cluster Admin</title>\"},{\"line\":58,\"text\":\"<h1>Homelab Cluster Admin</h1>\"},{\"line\":74,\"text\":\"<div class=\\\"card\\\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"}],\"units\":[{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"Group\":\"root\",\"LoadState\":\"loaded\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"root\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}]},\"v15_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"declared_report_bytes\":85056,\"declared_report_sha256\":\"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf\",\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":85940,\"output_sha256\":\"c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8\",\"output_source\":\"runner_json\",\"rollback_restored\":null,\"rollback_started\":false,\"runner_document_rc\":0,\"runner_json_sha256\":\"6769d194cb102deb6fc37293d887af601bd012448bac4de49dd4bca3301f34a7\",\"runner_rc\":0,\"runner_status\":\"OK\",\"runner_text_sha256\":\"c68484da49e0164adaccfe9ecf92120c53d9324b2c3dcb96b4088376d3d177d2\",\"runner_wrapper_matches_document\":true,\"sanitized_report_bytes\":85006,\"sanitized_report_sha256\":\"f05595551c37c00d473df29113cf9d597b2d3467ad5a2b50e8803e8579accc82\",\"sanitizer_byte_delta\":50,\"sanitizer_changed_report\":true,\"state_document_sha256\":\"f438ecd342026e3db3f910ff87aba2e3c746147cf3fa64217fc4317774055874\",\"state_path_sha256\":\"993095398c2c97878a69cd2e0aaa11e2ae1f587df8c8331835b5fd7ab7aa9de7\",\"state_status\":\"OK\"}}\n"
+  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z\",\"command_rc\":0,\"mutation_outcome\":\"NO_MUTATION\",\"output_truncated_in_json\":false,\"rc\":0,\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\n{\"api\":{\"fetch_failure_count\":0,\"fetched_file_count\":104,\"fetched_total_bytes\":297443,\"route_coverage\":{\"blob\":true,\"branch\":true,\"repo\":true,\"tree\":true},\"text_file_count\":104,\"tree\":{\"blob_count\":125,\"body_emitted\":false,\"complete\":true,\"entry_count\":161,\"exact_commit\":\"fe1221b06643db3b00a621b0230f92a46a4edc2b\",\"json_parse_ok\":true,\"status\":200,\"tree_count\":36,\"truncated\":false}},\"authorization_sent\":true,\"authorization_values_emitted\":false,\"branch\":{\"body_emitted\":false,\"branch\":\"main\",\"commit_sha\":\"fe1221b06643db3b00a621b0230f92a46a4edc2b\",\"json_parse_ok\":true,\"matches_expected_commit\":true,\"status\":200},\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z\",\"command_rc\":0,\"credential_values_emitted\":false,\"endpoint_summary\":{\"address_literal_occurrence_count\":81,\"raw_addresses_emitted\":false,\"raw_url_occurrence_count\":6,\"raw_urls_emitted\":false,\"scope_counts\":{\"external_hostname\":2,\"known_external_gitea\":1,\"private\":1},\"tokens\":[{\"host_scope\":\"external_hostname\",\"host_sha256\":\"8005522570737cc57f67406b7c6fb5d572797e4ba947dfb0bf3befee3639e667\",\"path_depth\":2,\"port\":null,\"scheme\":\"https\"},{\"host_scope\":\"known_external_gitea\",\"host_sha256\":\"92feea413ab6bd72fdd8b9102d872777d284d002a6be3e5323ca6e848105ee88\",\"path_depth\":2,\"port\":null,\"scheme\":\"https\"},{\"host_scope\":\"external_hostname\",\"host_sha256\":\"11d6a89894efa93b767088253526b02bac35874c1f63783507a3fb1d3b194e8a\",\"path_depth\":3,\"port\":null,\"scheme\":\"https\"},{\"host_scope\":\"private\",\"host_sha256\":\"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69\",\"path_depth\":1,\"port\":8081,\"scheme\":\"http\"}],\"unique_endpoint_token_count\":4},\"error_fingerprint\":\"dadccd2e354a457c337b9089047ec0a0918a5d3cccd5a188b83f8dce425a59ab\",\"error_registry\":{\"content_sha256\":\"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64\",\"duplicate_fingerprint_count\":0,\"duplicate_id_count\":0,\"duplicate_ids\":[],\"fingerprint_value_count\":0,\"fingerprint_values_emitted\":false,\"migratable_record_count\":0,\"migration_coverage_ratio\":0.0,\"path\":\"errors/regression-cases.json\",\"present\":true,\"prevention_evidence_count\":0,\"raw_records_emitted\":false,\"record_count\":59,\"status_counts\":{},\"unique_fingerprint_count\":0,\"unique_id_count\":59},\"fact_record_summary\":{\"domain_record_counts\":{\"access\":10,\"freshness\":25,\"identity\":16},\"emitted_record_count\":25,\"raw_record_values_emitted\":false,\"secret_hashes_emitted\":false,\"source_record_counts\":{\"inventory/access-paths.json\":9,\"inventory/cluster-nodes.json\":3,\"observed/cluster-guests.json\":11,\"observed/current-context.json\":2},\"top_fields\":[\"status\",\"id\",\"node\",\"type\",\"name\",\"vmid\",\"address\",\"service\"],\"unique_record_count\":25},\"fact_records\":[],\"fact_records_omitted_but_count_retained\":true,\"files_written\":false,\"http_methods\":[\"GET\"],\"mutation_blueprint\":{\"atomicity_requirements\":[\"read all exact source blobs again at the expected branch commit\",\"build the full candidate tree in a temporary directory\",\"validate JSON, JSONL, schemas, gate self-tests and repository tests\",\"create byte-for-byte backup objects for every modified blob\",\"create one commit against the expected parent commit\",\"verify the new commit tree and CI-visible files\",\"on any failure restore the exact previous branch state\"],\"code_root_choice\":{\"basis\":\"existing_top_level_code_or_directory\",\"existing_python_file_count\":2,\"path\":\"tools/pre_command_gate.py\",\"path_exists\":false,\"root\":\"tools\"},\"files_to_create_or_replace\":[\"docs/CLUSTER-HANDBOOK.md\",\"inventory/cluster-reference.json\",\"docs/ERROR-SYSTEM-V2.md\",\"schemas/error-record.schema.json\",\"errors/error-registry.jsonl\",\"tools/pre_command_gate.py\",\"tests/test_error_system_v2.py\"],\"files_to_update_conditionally\":[\".gitea/workflows/ci.yml\",\"docs/WORKFLOW.md\"],\"migration_requirements\":{\"append_runtime_failures_from_known_error_gate\":true,\"derive_fingerprint_only_from_sanitized_stable_fields\":true,\"mark_parallel_draft_superseded_only_after_validation\":true,\"preserve_all_existing_records\":true,\"require_prevention_test_for_each_active_record\":true},\"migration_source\":\"errors/regression-cases.json\",\"ready\":true,\"transaction_preconditions\":{\"branch\":\"main\",\"exact_source_sha256\":{\".gitea/workflows/ci.yml\":\"b078ac1b57e0846a90fd6acb1f3cb0866dbed0fb3bbde0d7070d29a344907681\",\"docs/WORKFLOW.md\":\"87a6f9a0960ee7240261138ecea122fd8fbd727d87bb0b27b02c103c3bc2f122\",\"errors/regression-cases.json\":\"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64\",\"schemas/change.schema.json\":\"57fbce7c6ca1f53c5ec2f8cd604d787f01019a2184b611e3b4406b467331d0af\",\"schemas/context.schema.json\":\"84c3c29984402b25c102575b54bdb82544844e455d3d2e115fefc5f42021e043\"},\"expected_branch_commit\":\"fe1221b06643db3b00a621b0230f92a46a4edc2b\",\"proposed_paths\":[{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"docs/CLUSTER-HANDBOOK.md\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"inventory/cluster-reference.json\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"docs/ERROR-SYSTEM-V2.md\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"schemas/error-record.schema.json\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"errors/error-registry.jsonl\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"tests/test_error_system_v2.py\"},{\"creation_precondition\":\"must_remain_absent\",\"exists_at_exact_commit\":false,\"path\":\"tools/pre_command_gate.py\"}],\"repository\":\"homelab-admin/homelab-ops\"}},\"mutation_outcome\":\"NO_MUTATION\",\"network_requests_made\":true,\"next_action\":\"RUN_ONE_IDEMPOTENT_GITEA_TRANSACTION_CREATE_CLUSTER_HANDBOOK_AND_ERROR_SYSTEM_V2\",\"output_truncated_in_json\":false,\"pre_command_known_error_gate\":{\"checked\":true,\"known_error_count\":28,\"known_error_ids\":[\"candidate-file-limit-600-20260805\",\"runner-wrapper-mismatch-after-success-20260805\",\"canonical-cluster-handbook-ambiguous-20260805\",\"pvesh-adapter-context-unreadable-20260805\",\"homelab-ops-bare-repository-not-found-local-20260805\",\"gitea-network-fallback-stage-not-emitted-20260805\",\"gitea-owner-hardcoded-homelab-admin-20260805\",\"gitea-git-auth-required-or-credential-unavailable-20260805\",\"gitea-process-self-match-payload-name-20260805\",\"sanitize-bool-attributeerror-20260805\",\"reader-v3-path-misclassified-as-route-20260805\",\"reader-v3-loopback-assumption-20260805\",\"reader-v3-capability-not-treated-as-secret-20260805\",\"reader-v3-systemd-unit-fed-to-python-ast-20260805\",\"reader-v3-helper-control-flow-not-traced-20260805\",\"reader-v3-post-capability-operation-not-derived-20260805\",\"reader-v3-detected-operations-not-tested-20260805\",\"reader-v3-manifest-global-not-bound-in-sandbox-20260805\",\"reader-v3-result-file-second-source-not-read-20260805\",\"reader-v3-negative-control-mixed-with-production-20260805\",\"reader-v3-manifest-schema-guessed-20260805\",\"reader-result-output-bound-after-compaction-20260805\",\"reader-v3-generation-record-shape-guessed-20260805\",\"reader-v3-result-file-assumed-standalone-json-20260805\",\"reader-v3-gitea-detail-marker-false-inventory-20260805\",\"reverse-proxy-runtime-not-visible-on-host-20260805\",\"gitea-public-owner-not-anonymously-discoverable-20260805\",\"exact-output-long-line-not-projectable-by-portal-view-20260805\"],\"previous_exact_identity_verified\":true},\"previous_exact_verification\":{\"bundle_bytes_ok\":true,\"bundle_sha256_ok\":true,\"capsule_member_match_count\":0,\"capsule_sha256_basis\":\"verified exact lane before command publication; optional bundle member count reported separately\",\"checks\":{\"changes_made_false\":true,\"command_id\":true,\"command_rc_zero\":true,\"contract_ok\":true,\"mutation_outcome_no_mutation\":true,\"projection_ok\":true,\"rollback_started_false\":true,\"runner_rc_zero\":true,\"runner_status_ok\":true},\"embedded_output_match_count\":1,\"exists\":true,\"identity_fully_verified\":true,\"member_count\":6,\"path_from_exact_result\":true,\"runner_json_match_count\":1,\"runner_text_match_count\":1,\"unsafe_member_rejected\":false},\"private_addresses_emitted\":false,\"raw_repository_contents_emitted\":false,\"rc\":0,\"read_only\":true,\"ready_for_idempotent_mutation\":true,\"regression_tests\":{\"endpoint_token_hides_host\":true,\"git_blob_oid_semantics\":true,\"known_error_gate_complete\":true,\"passed\":true,\"private_address_not_emitted\":true,\"safe_record_fingerprint_deterministic\":true,\"sanitize_bool_type_safe\":true,\"secret_value_redacted_before_record_hash\":true,\"strict_handbook_excludes_error_registry\":true},\"repository\":{\"archived\":false,\"default_branch\":\"main\",\"empty\":false,\"full_name\":\"homelab-admin/homelab-ops\",\"identity_safe\":true,\"mirror\":false,\"name\":\"homelab-ops\",\"owner\":\"homelab-admin\",\"private\":true,\"size\":3837},\"result_projection\":{\"budget_bytes\":24976,\"bytes\":24326,\"steps\":[\"source_summaries_24\",\"fact_records_32\",\"drop_json_shapes_and_limit_headings\",\"handbook_candidates_4\",\"omit_fact_records\"]},\"rollback_restored\":null,\"rollback_started\":false,\"root_cause_classification\":\"GITEA_CLUSTER_FACTS_AND_DOCUMENT_GAPS_EXACTLY_ANALYZED_MUTATION_READY\",\"schema\":1,\"secret_hashes_emitted\":false,\"source_summaries\":[{\"blob_sha\":\"7a8cb9e2dd5bc1235937aa211293b47ed4e26263\",\"bytes\":29485,\"content_sha256\":\"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64\",\"domains\":{\"access\":9,\"backup\":22,\"error_system\":38,\"freshness\":5,\"internal_external\":1,\"repair\":38,\"restore\":9,\"services\":8},\"headings\":[],\"json_parse_ok\":true,\"path\":\"errors/regression-cases.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":60},{\"blob_sha\":\"bad7037197f12ce9ce0b8be3b5723345699680b5\",\"bytes\":2289,\"content_sha256\":\"e07ccfac39f862c2bce2da0d31112e8e055ca9d3f23a01a8f33302863239203a\",\"domains\":{\"access\":1,\"backup\":0,\"error_system\":1,\"freshness\":1,\"internal_external\":2,\"repair\":0,\"restore\":0,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"changes/CR-2026-0018/regression-cases-draft.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"d072c291ee6b99466be0f4897e9d1308ee02e2b2\",\"bytes\":1134,\"content_sha256\":\"df400edc01d8201665c92e7434f8b0663eac0d34aab35e918abfb9bb873877e8\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":0,\"freshness\":2,\"internal_external\":0,\"repair\":1,\"restore\":0,\"services\":4},\"headings\":[\"Caller inventory requirements\",\"Search surfaces\",\"Required record\",\"Gate\"],\"json_parse_ok\":false,\"path\":\"changes/CR-2026-0018/caller-inventory-requirements.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"823d591344d8b8f67e213e2b890acdbd7bfc9864\",\"bytes\":211,\"content_sha256\":\"d3347712264b62c8768c952575574140c388df6d4d045c824ed9a8722bad1454\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":1,\"restore\":0,\"services\":0},\"headings\":[\"Changes\"],\"json_parse_ok\":false,\"path\":\"changes/README.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"c42de139cdf529611d50ad86291fb40df5bc0d09\",\"bytes\":1652,\"content_sha256\":\"ac5dc881dcd9fcb72644bc893e866f463c9c2b9b4544e32434e21b5d34168fb5\",\"domains\":{\"access\":9,\"backup\":0,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":0,\"restore\":0,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"inventory/access-paths.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"5a1fb00b587a813cee2a7bd98be451bd22173f5e\",\"bytes\":4952,\"content_sha256\":\"6b732f5ce3af0e8066d020bf10ee81b764b6d93ee5f08a9c8f17d0075eeced32\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":0,\"restore\":1,\"services\":22},\"headings\":[],\"json_parse_ok\":true,\"path\":\"observed/cluster-guests.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"3327f8b644b3f3e83774d5e7748a8dfa6b4f1ebf\",\"bytes\":958,\"content_sha256\":\"5e7236f3b8072ff6115ab5bc50ad448425882e86463e90d85df1a1c8eb502d65\",\"domains\":{\"access\":0,\"backup\":1,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":1,\"restore\":0,\"services\":2},\"headings\":[\"Архитектура управления\",\"Разделение ответственности\"],\"json_parse_ok\":false,\"path\":\"docs/ARCHITECTURE.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"3b034041ab628d15e74a7167b5545b366aeb4ad1\",\"bytes\":526,\"content_sha256\":\"87a6f9a0960ee7240261138ecea122fd8fbd727d87bb0b27b02c103c3bc2f122\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":0,\"freshness\":1,\"internal_external\":0,\"repair\":1,\"restore\":0,\"services\":0},\"headings\":[\"Change workflow\"],\"json_parse_ok\":false,\"path\":\"docs/WORKFLOW.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"f03826dcf6f8c6af4a8a78206c1b841ffaffd2ca\",\"bytes\":3020,\"content_sha256\":\"91c66d8ee6622d925174055599ad5ece6270c35f8d7d7967a26609f7e290c7e6\",\"domains\":{\"access\":0,\"backup\":1,\"error_system\":3,\"freshness\":4,\"internal_external\":0,\"repair\":1,\"restore\":1,\"services\":0},\"headings\":[\"Одна постоянная команда\",\"Что проверяется автоматически\",\"Правила безопасности\"],\"json_parse_ok\":false,\"path\":\"docs/skladchik-cookie-refresh.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":17},{\"blob_sha\":\"b4979003f9bb899f4a5de41ede47eacaeac48dee\",\"bytes\":2264,\"content_sha256\":\"e74454f528cdb5522e2c43d959472a04b7cc36e33da7aa5a5bd664fc79015344\",\"domains\":{\"access\":2,\"backup\":0,\"error_system\":0,\"freshness\":2,\"internal_external\":0,\"repair\":0,\"restore\":0,\"services\":1},\"headings\":[\"Skladchik Moderator Assistant\",\"Назначение\",\"Что планируется подготовить позднее\",\"Что сейчас не делать\"],\"json_parse_ok\":false,\"path\":\"docs/planned/skladchik-moderator-assistant.md\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"9586f62339cc657d18d033c104229458ebcadb58\",\"bytes\":1290,\"content_sha256\":\"cfb21fbf6ead4b3537b5f6f43e19602365b7b16adba5f317e48f8f19f5d40475\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":2,\"freshness\":0,\"internal_external\":1,\"repair\":1,\"restore\":0,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"policies/operating-policy.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":1},{\"blob_sha\":\"2032d3ebab4794625854b8c80d23ab72aac9998b\",\"bytes\":1436,\"content_sha256\":\"e06b7e6285eda2885ef913aa954a70abc7c1d892199a8b7cea2ada98b8edba34\",\"domains\":{\"access\":0,\"backup\":2,\"error_system\":3,\"freshness\":0,\"internal_external\":0,\"repair\":3,\"restore\":2,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"policies/cluster-cloud-quorum.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"8352f9a2024afea21049cd7700e460ee096c6c53\",\"bytes\":969,\"content_sha256\":\"57fbce7c6ca1f53c5ec2f8cd604d787f01019a2184b611e3b4406b467331d0af\",\"domains\":{\"access\":0,\"backup\":2,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":2,\"restore\":0,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"schemas/change.schema.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"7dc9476a8ca0227b4d5bf0cbd2c706c60af21119\",\"bytes\":539,\"content_sha256\":\"84c3c29984402b25c102575b54bdb82544844e455d3d2e115fefc5f42021e043\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":0,\"freshness\":0,\"internal_external\":0,\"repair\":0,\"restore\":0,\"services\":0},\"headings\":[],\"json_parse_ok\":true,\"path\":\"schemas/context.schema.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"3d6801bd6e7faefd1fc4579c41dc387f3dd1a9f0\",\"bytes\":318,\"content_sha256\":\"b078ac1b57e0846a90fd6acb1f3cb0866dbed0fb3bbde0d7070d29a344907681\",\"domains\":{\"access\":0,\"backup\":0,\"error_system\":1,\"freshness\":0,\"internal_external\":0,\"repair\":0,\"restore\":0,\"services\":1},\"headings\":[],\"json_parse_ok\":false,\"path\":\".gitea/workflows/ci.yml\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"68ae87029a1660255df02ff192e5b9206e9a6c05\",\"bytes\":13579,\"content_sha256\":\"d3108777e5094468aa3e633740ce8a56c90ca957a9a62932be896bbe784680b0\",\"domains\":{\"access\":22,\"backup\":23,\"error_system\":16,\"freshness\":3,\"internal_external\":1,\"repair\":22,\"restore\":10,\"services\":13},\"headings\":[],\"json_parse_ok\":true,\"path\":\"tasks/skladchik-reports-monitor-controlled-reauth-v1/task.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":43},{\"blob_sha\":\"6e22f5ca60d95965765c03ac1d012631aaae68b0\",\"bytes\":6750,\"content_sha256\":\"3a8fdfb6b4b64880012fcc3bc6fee4be6f101377310325541f9776de6d1623f7\",\"domains\":{\"access\":0,\"backup\":4,\"error_system\":16,\"freshness\":4,\"internal_external\":2,\"repair\":18,\"restore\":1,\"services\":28},\"headings\":[],\"json_parse_ok\":true,\"path\":\"tasks/homelab-logrotate-namespace-latch-clearance-v1/task.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"a1528e7cbc2b1027bfd7faa220bf1f096cdde1f5\",\"bytes\":3665,\"content_sha256\":\"666072447ba11d346772f62f274dadc974c4463e20f67e67f82f228f1fcf2ce8\",\"domains\":{\"access\":20,\"backup\":17,\"error_system\":1,\"freshness\":1,\"internal_external\":0,\"repair\":4,\"restore\":10,\"services\":7},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/rollback.sh\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"d2adf1bf345063da2b02040697f4358ba4ca4ae1\",\"bytes\":4667,\"content_sha256\":\"a49bcb32ac7212d7fb9768db8a277cb5525c873c84dd6fb633edae35dc6de532\",\"domains\":{\"access\":8,\"backup\":14,\"error_system\":9,\"freshness\":1,\"internal_external\":0,\"repair\":1,\"restore\":6,\"services\":10},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"c6f94aa845bc4cd8fc061e5e03e0fd7771a01bc6\",\"bytes\":7440,\"content_sha256\":\"636254d2526f0f62184d47cdb4c30c2831b11869c2000c557666f60b55d7691b\",\"domains\":{\"access\":13,\"backup\":24,\"error_system\":13,\"freshness\":0,\"internal_external\":0,\"repair\":12,\"restore\":4,\"services\":8},\"headings\":[],\"json_parse_ok\":true,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/task.json\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":2},{\"blob_sha\":\"9cc7d2fa3bc044d311500a5cf798273c79e39c35\",\"bytes\":4233,\"content_sha256\":\"ac0885c9762b99f360256ad3b3894f9cbcc4a8deab529a19ef264fe446bff029\",\"domains\":{\"access\":8,\"backup\":16,\"error_system\":8,\"freshness\":1,\"internal_external\":0,\"repair\":7,\"restore\":0,\"services\":11},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/verify.sh\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":0},{\"blob_sha\":\"774e2b0e591c9ada9aac9700a1b8427645284f1e\",\"bytes\":4665,\"content_sha256\":\"34de7dd0cc60c90c1b74e7a91334cb737d33c761882d1483d5b39f15b439c6f1\",\"domains\":{\"access\":13,\"backup\":19,\"error_system\":1,\"freshness\":1,\"internal_external\":0,\"repair\":1,\"restore\":0,\"services\":7},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":3},{\"blob_sha\":\"938db71d80350ac328ec90fe7bb5176a4d6d50ac\",\"bytes\":2867,\"content_sha256\":\"3b8687392163cf5b6f8b0874087320fe74ceaa5fee897dd0e7762d8c71476a1c\",\"domains\":{\"access\":4,\"backup\":4,\"error_system\":1,\"freshness\":0,\"internal_external\":0,\"repair\":8,\"restore\":11,\"services\":3},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tasks/skladchik-reports-monitor-controlled-reauth-v1/phases/rollback.sh\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":2},{\"blob_sha\":\"aefe7ac6860ec2107c281485b35c8cba1f1196ef\",\"bytes\":4846,\"content_sha256\":\"cfde805604d58d87d975584431aeb024bef21e863c4293d4726e5504415fe9bc\",\"domains\":{\"access\":0,\"backup\":6,\"error_system\":5,\"freshness\":2,\"internal_external\":1,\"repair\":3,\"restore\":9,\"services\":0},\"headings\":[],\"json_parse_ok\":false,\"path\":\"tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py\",\"raw_content_emitted\":false,\"secret_key_name_occurrence_count\":21}],\"status\":\"OK\",\"strict_handbook\":{\"candidates\":[{\"canonical_phrase_hits\":0,\"content_sha256\":\"91c66d8ee6622d925174055599ad5ece6270c35f8d7d7967a26609f7e290c7e6\",\"covered_domains\":[\"backup\",\"error_system\",\"repair\",\"restore\"],\"headings\":[\"Одна постоянная команда\",\"Что проверяется автоматически\",\"Правила безопасности\"],\"path\":\"docs/skladchik-cookie-refresh.md\",\"score\":80,\"title_signal\":false},{\"canonical_phrase_hits\":1,\"content_sha256\":\"5e7236f3b8072ff6115ab5bc50ad448425882e86463e90d85df1a1c8eb502d65\",\"covered_domains\":[\"backup\",\"repair\",\"services\"],\"headings\":[\"Архитектура управления\",\"Разделение ответственности\"],\"path\":\"docs/ARCHITECTURE.md\",\"score\":75,\"title_signal\":false},{\"canonical_phrase_hits\":1,\"content_sha256\":\"d9cb129e009176a7dde2a07f59760acddd9e17237eef563e97a55b3e62838bd1\",\"covered_domains\":[\"repair\"],\"headings\":[\"homelab-ops\",\"Порядок работы\",\"Локальная проверка\"],\"path\":\"README.md\",\"score\":55,\"title_signal\":true},{\"canonical_phrase_hits\":0,\"content_sha256\":\"e74454f528cdb5522e2c43d959472a04b7cc36e33da7aa5a5bd664fc79015344\",\"covered_domains\":[\"access\",\"services\"],\"headings\":[\"Skladchik Moderator Assistant\",\"Назначение\",\"Что планируется подготовить позднее\",\"Что сейчас не делать\",\"Условие возобновления\"],\"path\":\"docs/planned/skladchik-moderator-assistant.md\",\"score\":40,\"title_signal\":false}],\"path\":null,\"previous_loose_candidate_rejected\":\"errors/regression-cases.json\",\"qualified_count\":0,\"rejection_reason\":\"error registry is not a cluster handbook under the strict source-class definition\",\"status\":\"NOT_FOUND\",\"strict_definition\":{\"allowed_source_classes\":[\"docs\",\"inventory\",\"root index\"],\"excluded_source_classes\":[\"errors\",\"changes\",\"tasks\",\"tests\",\"workflows\",\"schemas\"],\"minimum_domains\":6,\"requires_canonical_phrase\":true,\"requires_cluster_title\":true}},\"verified_gaps\":[{\"evidence\":{\"qualified_count\":0,\"strict_status\":\"NOT_FOUND\"},\"gap_id\":\"canonical_cluster_handbook_absent\",\"remediation\":\"create one canonical docs/CLUSTER-HANDBOOK.md and link every fact to an exact machine-readable source\",\"severity\":\"BLOCKING\"},{\"evidence\":{\"path_verified_absent\":true},\"gap_id\":\"machine_readable_cluster_reference_absent\",\"remediation\":\"create inventory/cluster-reference.json with source_sha256, verified_at and stale/unknown states\",\"severity\":\"BLOCKING\"},{\"evidence\":{\"record_count\":0},\"gap_id\":\"structured_backup_restore_facts_absent\",\"remediation\":\"materialize backup_restore facts into the machine-readable cluster reference\",\"severity\":\"HIGH\"},{\"evidence\":{\"record_count\":0},\"gap_id\":\"structured_repair_facts_absent\",\"remediation\":\"materialize repair facts into the machine-readable cluster reference\",\"severity\":\"HIGH\"},{\"evidence\":{\"record_count\":0},\"gap_id\":\"structured_scope_facts_absent\",\"remediation\":\"materialize scope facts into the machine-readable cluster reference\",\"severity\":\"HIGH\"},{\"evidence\":{\"error_record_count\":59,\"fingerprint_count\":0},\"gap_id\":\"error_registry_has_no_fingerprints\",\"remediation\":\"seed stable normalized fingerprints and block duplicate failed fingerprints before publication\",\"severity\":\"BLOCKING\"},{\"evidence\":{\"error_record_count\":59,\"prevention_evidence_count\":0},\"gap_id\":\"error_registry_has_no_prevention_evidence\",\"remediation\":\"require one positive and one negative prevention test for every active error record\",\"severity\":\"BLOCKING\"},{\"evidence\":{\"ci_mentions_fingerprint\":false,\"ci_mentions_pre_command_gate\":false},\"gap_id\":\"ci_does_not_enforce_error_fingerprint_gate\",\"remediation\":\"wire the pre-command gate and registry/schema tests into .gitea/workflows/ci.yml\",\"severity\":\"BLOCKING\"},{\"evidence\":{\"both_present\":true,\"content_sha256_equal\":false},\"gap_id\":\"parallel_draft_and_canonical_error_registries_diverge\",\"remediation\":\"declare canonical ownership and mark the draft superseded after migration\",\"severity\":\"MEDIUM\"},{\"evidence\":{\"matching_path_count\":0},\"gap_id\":\"superseded_command_registry_absent\",\"remediation\":\"record superseded command_id/capsule/result relationships during post-result ingestion\",\"severity\":\"HIGH\"}],\"workflow_gate_signals\":{\"ci_mentions_error_registry\":false,\"ci_mentions_fingerprint\":false,\"ci_mentions_pre_command_gate\":false,\"ci_present\":true,\"repository_mentions_duplicate_failed_command_blocked\":false,\"repository_mentions_error_register_checked\":false,\"repository_mentions_reference_register_checked\":false}}\n"
 }
diff --git a/runtime/latest.txt b/runtime/latest.txt
index 6db810c..687b9d6 100644
--- a/runtime/latest.txt
+++ b/runtime/latest.txt
@@ -1,15 +1,15 @@
 CHAT_OUTPUT_BEGIN
-COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z
+COMMAND_ID=HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z
 STATUS=OK
 RC=0
 HOST=pve01
 MODE=verify
-COMPONENT=homelab-cluster-admin-update-visibility
+COMPONENT=cluster-knowledge-base-error-system
 REFERENCE_REGISTER_CHECK=OK
 REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
 ERROR_REGISTER_CHECK=OK
 ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
-COMMAND_SHA256=8e9116b7bd5174aea8762e6a52c7c3efb4eb6ea1c0e202c3bae875aaead38778
+COMMAND_SHA256=9088e8f5ce54b2b810894f0d1860a25390b3179b29d85f63e19e6f81c9f066a6
 DUPLICATE_FAILED_COMMAND_BLOCKED=false
 EXECUTION_STARTED=true
 CHANGE_DECLARED=false
@@ -24,11 +24,11 @@ MUTATION_OUTCOME=NO_MUTATION
 SANITIZED=yes
 SECRETS_INCLUDED=no
 PRIVATE_ADDRESSES_INCLUDED=no
-RAW_EVIDENCE_SHA256=7789c8f8591d0be49d972a2812151287eb12642bd99740576b1b9d9a29f84267
-SANITIZED_OUTPUT_SHA256=5b17a087bd51a9f4e901cd820e129160e70081636d2da63d902f481305df4f99
+RAW_EVIDENCE_SHA256=6c2d4b05d4a1d95e2dfbc5ebcff23469e469ec2d5e1550a49d07c3ad51a5f205
+SANITIZED_OUTPUT_SHA256=6c2d4b05d4a1d95e2dfbc5ebcff23469e469ec2d5e1550a49d07c3ad51a5f205
 OUTPUT_BEGIN
-HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z","command_rc":0,"control_plane_error":null,"exact_source_count":30,"index_source_count":1,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_atomic_transaction":true,"reference_source_count":1,"remote_probe_privilege":"sudo-root","report_bytes":193466,"report_sha256":"772f2c3f77d7909de34aa159e0e2c8c2ca5480ff51cbcd4bcb194af9019b43d0","rollback_restored":false,"rollback_started":false,"status":"OK","unit_record_count":16,"v15_declared_report_sha256":"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf","v15_exact_result_identity_confirmed":true,"v15_output_sha256":"c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8","v15_sanitized_report_sha256":"f05595551c37c00d473df29113cf9d597b2d3467ad5a2b50e8803e8579accc82","v15_sanitizer_byte_delta":50,"version":1}
-{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V18-REMOTE-EXACT-MAP-READ-ONLY-20260805T132500Z","mutation_outcome":"NO_MUTATION","read_only":true,"remote_exact_map":{"euid":0,"exact_manifest":[{"bytes":3708,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","priority":100,"requested_as":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","uid":0},{"bytes":10353,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","priority":95,"requested_as":"/var/www/homelab-cluster-admin/index.html","sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","uid":0},{"bytes":2832,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","priority":92,"requested_as":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5","uid":0},{"bytes":3471,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","priority":92,"requested_as":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e","uid":0},{"bytes":1433,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","priority":92,"requested_as":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792","uid":0},{"bytes":848,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","priority":92,"requested_as":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972","uid":0},{"error":"RuntimeError: unsafe file: /usr/bin/python3.11","is_dir":false,"is_file":true,"is_symlink":false,"path":"/usr/bin/python3.11","priority":92,"requested_as":"/usr/bin/python3"},{"bytes":2541,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/usr/local/sbin/homelab-stage4i-verify","priority":92,"requested_as":"/usr/local/sbin/homelab-stage4i-verify","sha256":"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d","uid":0},{"bytes":616,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","priority":90,"requested_as":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7","uid":0},{"bytes":231,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","priority":90,"requested_as":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae","uid":0},{"bytes":803,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","priority":90,"requested_as":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea","uid":0},{"bytes":1190,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","priority":90,"requested_as":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771","uid":0},{"bytes":223,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","priority":90,"requested_as":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d","uid":0},{"bytes":133,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b","uid":0},{"bytes":154,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23","uid":0},{"bytes":131,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042","uid":0},{"bytes":149,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b","uid":0},{"bytes":127,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d","uid":0},{"bytes":151,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1","uid":0},{"bytes":138,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e","uid":0},{"bytes":154,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311","uid":0},{"bytes":301,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","priority":90,"requested_as":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","uid":0},{"bytes":465,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.service","priority":90,"requested_as":"/etc/systemd/system/homelab-stage4i-verify.service","sha256":"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b","uid":0},{"bytes":224,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.timer","priority":90,"requested_as":"/etc/systemd/system/homelab-stage4i-verify.timer","sha256":"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396","uid":0},{"is_dir":true,"is_file":false,"is_symlink":false,"path":"/opt/cluster-admin-incident-engine","priority":75,"requested_as":"/opt/cluster-admin-incident-engine"},{"bytes":380,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","priority":70,"requested_as":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e","uid":0},{"bytes":217,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","priority":70,"requested_as":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053","uid":0},{"bytes":128,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","priority":70,"requested_as":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac","uid":0},{"bytes":1535,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","priority":70,"requested_as":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b","uid":0},{"bytes":616,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","priority":65,"requested_as":"nearby","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7","uid":0},{"bytes":231,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","priority":65,"requested_as":"nearby","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae","uid":0},{"bytes":803,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","priority":65,"requested_as":"nearby","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea","uid":0},{"bytes":1190,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","priority":65,"requested_as":"nearby","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771","uid":0},{"bytes":223,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","priority":65,"requested_as":"nearby","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d","uid":0},{"bytes":133,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","priority":65,"requested_as":"nearby","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b","uid":0},{"bytes":154,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","priority":65,"requested_as":"nearby","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23","uid":0},{"bytes":131,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","priority":65,"requested_as":"nearby","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042","uid":0},{"bytes":149,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","priority":65,"requested_as":"nearby","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b","uid":0},{"bytes":127,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","priority":65,"requested_as":"nearby","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d","uid":0},{"bytes":151,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","priority":65,"requested_as":"nearby","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1","uid":0},{"bytes":138,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","priority":65,"requested_as":"nearby","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e","uid":0},{"bytes":154,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","priority":65,"requested_as":"nearby","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311","uid":0},{"bytes":301,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","priority":65,"requested_as":"nearby","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","uid":0},{"bytes":465,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.service","priority":65,"requested_as":"nearby","sha256":"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b","uid":0},{"bytes":224,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.timer","priority":65,"requested_as":"nearby","sha256":"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396","uid":0},{"bytes":2832,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","priority":65,"requested_as":"nearby","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5","uid":0},{"bytes":3471,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","priority":65,"requested_as":"nearby","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e","uid":0},{"bytes":1433,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","priority":65,"requested_as":"nearby","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792","uid":0},{"bytes":3708,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","priority":65,"requested_as":"nearby","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","uid":0},{"bytes":848,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","priority":65,"requested_as":"nearby","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972","uid":0},{"bytes":10353,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","priority":65,"requested_as":"nearby","sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","uid":0},{"bytes":89,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","priority":50,"requested_as":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f","uid":0},{"bytes":28599,"gid":994,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6","uid":0},{"bytes":10640,"gid":994,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe","uid":0},{"bytes":1857,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670","uid":0},{"bytes":2355,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588","uid":0},{"bytes":1342,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c","uid":0},{"bytes":922,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","priority":50,"requested_as":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3","uid":0},{"bytes":8444,"gid":994,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","priority":50,"requested_as":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a","uid":0},{"bytes":131,"gid":994,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","priority":50,"requested_as":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"5348849c615d0258d7812fdcf660611f911b97753bbab2e0fd87e5b1c2a98cf8","uid":999},{"bytes":259,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"275b09782748c2b8a1b762b6390a84df202e478ece7e8f377788569b2cc5b027","uid":0},{"bytes":168,"gid":994,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"5b2bc5ae640f3b7b42000bae6b5b63d76ba00a8f55f4b6cc1cc9f80871373653","uid":999},{"bytes":189,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941","uid":0},{"bytes":141,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6","uid":0},{"bytes":199,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680","uid":0},{"bytes":272,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"49dbf42b944b78c7394d299a08400d24a92b7532fe8d9774c686fe6a59972e15","uid":0},{"bytes":134,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"6353b738b98d4d038684bc753014410f96cf0ace91227dad461b4a2c69ee3c29","uid":0},{"bytes":214,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","priority":50,"requested_as":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"02916ca8a08ee305709d7fa7935209a6321b0e25fb105e0f2d3e242ed66fb0f0","uid":0},{"bytes":214,"gid":0,"is_dir":false,"is_file":true,"is_symlink":false,"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","priority":50,"requested_as":"/var/www/homelab-cluster-admin/status.txt","sha256":"02916ca8a08ee305709d7fa7935209a6321b0e25fb105e0f2d3e242ed66fb0f0","uid":0}],"exact_sources":[{"bytes":3708,"line_count":79,"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","priority":100,"sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":5,"text":"mkdir -p \"$WEB\""},{"line":6,"text":"BAD=0"},{"line":7,"text":""},{"line":8,"text":"line_or_missing() {"},{"line":9,"text":"  f=\"$1\""},{"line":10,"text":"  if [ -s \"$f\" ]; then head -n1 \"$f\"; else echo \"STATUS=WARN TYPE=missing FILE=$f\"; fi"},{"line":11,"text":"}"},{"line":12,"text":""},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":20,"text":"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":21,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":22,"text":""},{"line":23,"text":"for line in \"$SELF\" \"$OBSERVER\" \"$FULL\" \"$BACKUP\"; do"},{"line":24,"text":"  echo \"$line\" | grep -q '^STATUS=OK' || BAD=$((BAD+1))"},{"line":25,"text":"done"},{"line":26,"text":"echo \"$FULL\" | grep -q 'BAD=0' || BAD=$((BAD+1))"},{"line":27,"text":"echo \"$FULL\" | grep -q 'WARN=0' || BAD=$((BAD+1))"},{"line":28,"text":"echo \"$BACKUP\" | grep -q 'BAD=0' || BAD=$((BAD+1))"},{"line":29,"text":"echo \"$BACKUP\" | grep -q 'MAIL_CLOUD_2COPY_VERIFY_OK' || BAD=$((BAD+1))"},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":31,"text":"[ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":32,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":33,"text":""},{"line":34,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; COLOR=\"#0a7f38\"; BADGE=\"OK\"; else STATUS=WARN; COLOR=\"#b26a00\"; BADGE=\"REVIEW\"; fi"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":37,"text":"echo \"$STATUS_LINE\" > \"$WEB/status.txt\""},{"line":38,"text":""},{"line":39,"text":"cat > \"$WEB/index.html\" <<HTML"},{"line":40,"text":"<!doctype html>"},{"line":41,"text":"<html lang=\"ru\">"},{"line":42,"text":"<head>"},{"line":43,"text":"<meta charset=\"utf-8\">"},{"line":44,"text":"<meta http-equiv=\"refresh\" content=\"60\">"},{"line":45,"text":"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":47,"text":"<style>"},{"line":48,"text":"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}"},{"line":49,"text":".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}"},{"line":50,"text":"h1{margin:0 0 8px}"},{"line":51,"text":".badge{display:inline-block;background:$COLOR;color:white;padding:8px 14px;border-radius:999px;font-weight:700}"},{"line":52,"text":"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}"},{"line":53,"text":".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}"},{"line":54,"text":".small{color:#aab2c0}"},{"line":55,"text":"</style>"},{"line":56,"text":"</head>"},{"line":57,"text":"<body>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":59,"text":"<div class=\"small\">generated UTC: $TS · VM180 · [PRIVATE_IP] · refresh 60s</div>"},{"line":60,"text":"<p><span class=\"badge\">$BADGE</span></p>"},{"line":61,"text":""},{"line":62,"text":"<div class=\"grid\">"},{"line":63,"text":"<div class=\"card\"><h2>Self</h2><pre>$SELF"},{"line":64,"text":"FAILED_UNITS=$FAILED"},{"line":65,"text":"QGA=$QGA</pre></div>"},{"line":66,"text":"<div class=\"card\"><h2>Observer</h2><pre>$OBSERVER</pre></div>"},{"line":67,"text":"<div class=\"card\"><h2>Full observer</h2><pre>$FULL</pre></div>"},{"line":68,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>$BACKUP</pre></div>"},{"line":69,"text":"</div>"},{"line":70,"text":""},{"line":71,"text":"<div class=\"card\"><h2>Overall</h2><pre>$OVERALL</pre></div>"},{"line":72,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>$FINAL</pre></div>"},{"line":73,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>$STRICT</pre></div>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":75,"text":"</body>"},{"line":76,"text":"</html>"},{"line":77,"text":"HTML"},{"line":78,"text":""},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}]},{"bytes":10353,"line_count":80,"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","priority":95,"sha256":"c4fc9eec9c1e9884c5b054b2193db518f5a5d15c026c5e01fe11c6be8d52fc89","source":[{"line":1,"text":"<!doctype html>"},{"line":2,"text":"<html lang=\"ru\">"},{"line":3,"text":"<head>"},{"line":4,"text":"<meta charset=\"utf-8\">"},{"line":5,"text":"<meta http-equiv=\"refresh\" content=\"60\">"},{"line":6,"text":"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"},{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":8,"text":"<style>"},{"line":9,"text":"body{font-family:system-ui,-apple-system,Segoe UI,Arial,sans-serif;background:#0f1115;color:#e8e8e8;margin:0;padding:24px}"},{"line":10,"text":".card{background:#181c24;border:1px solid #2a3140;border-radius:14px;padding:18px;margin:14px 0;box-shadow:0 8px 24px rgba(0,0,0,.22)}"},{"line":11,"text":"h1{margin:0 0 8px}"},{"line":12,"text":".badge{display:inline-block;background:#b26a00;color:white;padding:8px 14px;border-radius:999px;font-weight:700}"},{"line":13,"text":"pre{white-space:pre-wrap;word-break:break-word;background:#0b0d11;border-radius:10px;padding:12px;overflow:auto}"},{"line":14,"text":".grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px}"},{"line":15,"text":".small{color:#aab2c0}"},{"line":16,"text":"</style>"},{"line":17,"text":"</head>"},{"line":18,"text":"<body>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":20,"text":"<div class=\"small\">generated UTC: 2026-08-05T13:20:48Z · VM180 · [PRIVATE_IP] · refresh 60s</div>"},{"line":21,"text":"<p><span class=\"badge\">REVIEW</span></p>"},{"line":22,"text":""},{"line":23,"text":"<div class=\"grid\">"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":25,"text":"FAILED_UNITS=1"},{"line":26,"text":"QGA=active</pre></div>"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":30,"text":"</div>"},{"line":31,"text":""},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":36,"text":"generated_utc=2026-08-05T13:19:08Z"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":38,"text":""},{"line":39,"text":"## Restricted probes"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:10Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T13:19:12Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T13:19:14Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":44,"text":""},{"line":45,"text":"## Service map checks"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":47,"text":"OK severity=critical kind=tcp name=pve01_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":48,"text":"OK severity=critical kind=tcp name=pve01_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0"},{"line":49,"text":"OK severity=critical kind=tcp name=pve01_health target=[PRIVATE_IP] value=9101 tcp_9101_rc=0"},{"line":50,"text":"OK severity=critical kind=tcp name=pve02_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":51,"text":"OK severity=critical kind=tcp name=pve02_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0"},{"line":52,"text":"OK severity=critical kind=tcp name=pve03_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":53,"text":"OK severity=critical kind=tcp name=pve03_proxmox target=[PRIVATE_IP] value=8006 tcp_8006_rc=0"},{"line":54,"text":"OK severity=critical kind=tcp name=dns1_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0"},{"line":55,"text":"OK severity=critical kind=tcp name=dns2_tcp53 target=[PRIVATE_IP] value=53 tcp_53_rc=0"},{"line":56,"text":"OK severity=critical kind=tcp name=edge_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":58,"text":"OK severity=critical kind=ping name=vm130_edge_vm_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":59,"text":"OK severity=critical kind=tcp name=vm130_edge_vm_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":60,"text":"OK severity=critical kind=ping name=vm150_nextcloud_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":61,"text":"OK severity=critical kind=tcp name=vm150_nextcloud_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":62,"text":"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1"},{"line":63,"text":"INFO_OFF severity=optional kind=tcp name=vm150_nextcloud_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":64,"text":"OK severity=critical kind=ping name=vm160_forum_prod_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":65,"text":"OK severity=critical kind=tcp name=vm160_forum_prod_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":66,"text":"OK severity=optional kind=tcp name=vm160_forum_prod_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=0"},{"line":67,"text":"INFO_OFF severity=optional kind=tcp name=vm160_forum_prod_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":68,"text":"OK severity=critical kind=ping name=vm170_core_apps_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":69,"text":"OK severity=critical kind=tcp name=vm170_core_apps_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":70,"text":"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1"},{"line":71,"text":"INFO_OFF severity=optional kind=tcp name=vm170_core_apps_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":72,"text":"OK severity=critical kind=ping name=vm171_monitoring_ping target=[PRIVATE_IP] value=- ping_rc=0"},{"line":73,"text":"OK severity=critical kind=tcp name=vm171_monitoring_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":74,"text":"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_http80 target=[PRIVATE_IP] value=80 tcp_80_rc=1"},{"line":75,"text":"INFO_OFF severity=optional kind=tcp name=vm171_monitoring_https443 target=[PRIVATE_IP] value=443 tcp_443_rc=1"},{"line":76,"text":""},{"line":77,"text":"## Result"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T13:19:08Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":79,"text":"</body>"},{"line":80,"text":"</html>"}]},{"bytes":2832,"line_count":96,"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","priority":92,"sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":4,"text":"exec 9>\"$LOCK\""},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":6,"text":""},{"line":7,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":10,"text":"REPORT=\"$ROOT/latest.md\""},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":12,"text":"TMP=\"$REPORT.tmp\""},{"line":13,"text":"mkdir -p \"$ROOT\" \"$HEALTH_DIR\""},{"line":14,"text":"BAD=0"},{"line":15,"text":"WARN=0"},{"line":16,"text":""},{"line":17,"text":"check_ping() {"},{"line":18,"text":"  name=\"$1\"; ip=\"$2\""},{"line":19,"text":"  ping -c1 -W1 \"$ip\" >/dev/null 2>&1"},{"line":20,"text":"  rc=$?"},{"line":21,"text":"  echo \"$name ping rc=$rc\" >> \"$TMP\""},{"line":22,"text":"  [ \"$rc\" = 0 ] || BAD=$((BAD+1))"},{"line":23,"text":"}"},{"line":24,"text":""},{"line":25,"text":"check_tcp() {"},{"line":26,"text":"  name=\"$1\"; ip=\"$2\"; port=\"$3\""},{"line":27,"text":"  timeout 2 bash -c \"</dev/tcp/$ip/$port\" >/dev/null 2>&1"},{"line":28,"text":"  rc=$?"},{"line":29,"text":"  echo \"$name tcp/$port rc=$rc\" >> \"$TMP\""},{"line":30,"text":"  [ \"$rc\" = 0 ] || BAD=$((BAD+1))"},{"line":31,"text":"}"},{"line":32,"text":""},{"line":33,"text":"check_http_status_ok() {"},{"line":34,"text":"  name=\"$1\"; url=\"$2\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":36,"text":"  rc=$?"},{"line":37,"text":"  echo \"$name http rc=$rc line=$line\" >> \"$TMP\""},{"line":38,"text":"  echo \"$line\" | grep -q \"^STATUS=\""},{"line":39,"text":"  okrc=$?"},{"line":40,"text":"  [ \"$rc\" = 0 ] && [ \"$okrc\" = 0 ] || BAD=$((BAD+1))"},{"line":41,"text":"}"},{"line":42,"text":""},{"line":43,"text":"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":44,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":46,"text":""},{"line":47,"text":"{"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":49,"text":"  echo \"generated_utc=$TS\""},{"line":50,"text":"  echo"},{"line":51,"text":"  echo \"## Self\""},{"line":52,"text":"  echo \"host=$(hostname 2>/dev/null)\""},{"line":53,"text":"  echo \"failed_units=$FAILED\""},{"line":54,"text":"  echo \"qga=$QGA\""},{"line":55,"text":"  echo \"self_check=$SELF_LINE\""},{"line":56,"text":"  echo"},{"line":57,"text":"  echo \"## Network checks\""},{"line":58,"text":"} > \"$TMP\""},{"line":59,"text":""},{"line":60,"text":"[ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":61,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":62,"text":"echo \"$SELF_LINE\" | grep -q \"^STATUS=OK\" || BAD=$((BAD+1))"},{"line":63,"text":""},{"line":64,"text":"check_ping pve01 [PRIVATE_IP]"},{"line":65,"text":"check_tcp pve01_ssh [PRIVATE_IP] 22"},{"line":66,"text":"check_tcp pve01_proxmox [PRIVATE_IP] 8006"},{"line":67,"text":"check_tcp pve01_health [PRIVATE_IP] 9101"},{"line":68,"text":""},{"line":69,"text":"check_ping pve02 [PRIVATE_IP]"},{"line":70,"text":"check_tcp pve02_ssh [PRIVATE_IP] 22"},{"line":71,"text":"check_tcp pve02_proxmox [PRIVATE_IP] 8006"},{"line":72,"text":""},{"line":73,"text":"check_ping pve03 [PRIVATE_IP]"},{"line":74,"text":"check_tcp pve03_ssh [PRIVATE_IP] 22"},{"line":75,"text":"check_tcp pve03_proxmox [PRIVATE_IP] 8006"},{"line":76,"text":""},{"line":77,"text":"check_ping edge [PRIVATE_IP]"},{"line":78,"text":"check_tcp edge_ssh [PRIVATE_IP] 22"},{"line":79,"text":""},{"line":80,"text":"check_ping dns1 [PRIVATE_IP]"},{"line":81,"text":"check_tcp dns1_tcp53 [PRIVATE_IP] 53"},{"line":82,"text":"check_ping dns2 [PRIVATE_IP]"},{"line":83,"text":"check_tcp dns2_tcp53 [PRIVATE_IP] 53"},{"line":84,"text":""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":86,"text":""},{"line":87,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""},{"line":89,"text":"echo \"$LINE\" | tee \"$HEALTH\" >/dev/null"},{"line":90,"text":"{"},{"line":91,"text":"  echo"},{"line":92,"text":"  echo \"## Result\""},{"line":93,"text":"  echo \"$LINE\""},{"line":94,"text":"} >> \"$TMP\""},{"line":95,"text":"mv \"$TMP\" \"$REPORT\""},{"line":96,"text":"cat \"$HEALTH\""}]},{"bytes":3471,"line_count":113,"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","priority":92,"sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":7,"text":"REPORT=\"$ROOT/full-observer-latest.md\""},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":10,"text":"mkdir -p \"$ROOT\" \"$HEALTH_DIR\""},{"line":11,"text":"exec 9>\"$LOCK\""},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":13,"text":""},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":15,"text":"BAD=0"},{"line":16,"text":"WARN=0"},{"line":17,"text":"INFO_OFF=0"},{"line":18,"text":"TOTAL=0"},{"line":19,"text":"CRIT_FAIL=0"},{"line":20,"text":"OPT_OFF=0"},{"line":21,"text":"PROBE_OK=0"},{"line":22,"text":"PROBE_BAD=0"},{"line":23,"text":""},{"line":24,"text":"TMP=\"$REPORT.tmp\""},{"line":25,"text":"{"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":27,"text":"  echo \"generated_utc=$TS\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":29,"text":"  echo"},{"line":30,"text":"  echo \"## Restricted probes\""},{"line":31,"text":"} > \"$TMP\""},{"line":32,"text":""},{"line":33,"text":"for t in \"root@[PRIVATE_IP] pve01\" \"root@[PRIVATE_IP] pve02\" \"root@[PRIVATE_IP] pve03\" \"debian@[PRIVATE_IP] edge\"; do"},{"line":34,"text":"  set -- $t"},{"line":35,"text":"  target=\"$1\"; name=\"$2\""},{"line":36,"text":"  out=$(ssh -n -i \"$K\" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 \"$target\" probe 2>/tmp/full-probe-$name.err)"},{"line":37,"text":"  rc=$?"},{"line":38,"text":"  echo \"$name ssh_rc=$rc $out\" >> \"$TMP\""},{"line":39,"text":"  echo \"$out\" | grep -q '^STATUS=OK'"},{"line":40,"text":"  ok=$?"},{"line":41,"text":"  if [ \"$rc\" = 0 ] && [ \"$ok\" = 0 ]; then PROBE_OK=$((PROBE_OK+1)); else PROBE_BAD=$((PROBE_BAD+1)); BAD=$((BAD+1)); fi"},{"line":42,"text":"done"},{"line":43,"text":""},{"line":44,"text":"{"},{"line":45,"text":"  echo"},{"line":46,"text":"  echo \"## Service map checks\""},{"line":47,"text":"} >> \"$TMP\""},{"line":48,"text":""},{"line":49,"text":"while IFS=\"$(printf '\\t')\" read -r severity kind name target value; do"},{"line":50,"text":"  [ -n \"$severity\" ] || continue"},{"line":51,"text":"  TOTAL=$((TOTAL+1))"},{"line":52,"text":"  rc=0"},{"line":53,"text":"  detail=\"\""},{"line":54,"text":"  case \"$kind\" in"},{"line":55,"text":"    ping)"},{"line":56,"text":"      ping -c1 -W1 \"$target\" >/dev/null 2>&1"},{"line":57,"text":"      rc=$?"},{"line":58,"text":"      detail=\"ping_rc=$rc\""},{"line":59,"text":"      ;;"},{"line":60,"text":"    tcp)"},{"line":61,"text":"      timeout 2 bash -c \"</dev/tcp/$target/$value\" >/dev/null 2>&1"},{"line":62,"text":"      rc=$?"},{"line":63,"text":"      detail=\"tcp_${value}_rc=$rc\""},{"line":64,"text":"      ;;"},{"line":65,"text":"    http_status_ok)"},{"line":66,"text":"      line=$(curl -fsS --max-time 5 \"$value\" 2>/tmp/full-http-$name.err | head -n1)"},{"line":67,"text":"      curlrc=$?"},{"line":68,"text":"      echo \"$line\" | grep -q '^STATUS='"},{"line":69,"text":"      okrc=$?"},{"line":70,"text":"      [ \"$curlrc\" = 0 ] && [ \"$okrc\" = 0 ]"},{"line":71,"text":"      rc=$?"},{"line":72,"text":"      detail=\"http_rc=$curlrc ok_rc=$okrc line=$line\""},{"line":73,"text":"      ;;"},{"line":74,"text":"    missing)"},{"line":75,"text":"      rc=1"},{"line":76,"text":"      detail=\"missing_target=$target value=$value\""},{"line":77,"text":"      ;;"},{"line":78,"text":"    *)"},{"line":79,"text":"      rc=1"},{"line":80,"text":"      detail=\"unknown_kind=$kind\""},{"line":81,"text":"      ;;"},{"line":82,"text":"  esac"},{"line":83,"text":""},{"line":84,"text":"  if [ \"$rc\" = 0 ]; then"},{"line":85,"text":"    echo \"OK severity=$severity kind=$kind name=$name target=$target value=$value $detail\" >> \"$TMP\""},{"line":86,"text":"  else"},{"line":87,"text":"    if [ \"$severity\" = critical ]; then"},{"line":88,"text":"      echo \"FAIL severity=critical kind=$kind name=$name target=$target value=$value $detail\" >> \"$TMP\""},{"line":89,"text":"      CRIT_FAIL=$((CRIT_FAIL+1))"},{"line":90,"text":"      BAD=$((BAD+1))"},{"line":91,"text":"    else"},{"line":92,"text":"      echo \"INFO_OFF severity=optional kind=$kind name=$name target=$target value=$value $detail\" >> \"$TMP\""},{"line":93,"text":"      OPT_OFF=$((OPT_OFF+1))"},{"line":94,"text":"      INFO_OFF=$((INFO_OFF+1))"},{"line":95,"text":"    fi"},{"line":96,"text":"  fi"},{"line":97,"text":"done < \"$MAP\""},{"line":98,"text":""},{"line":99,"text":"SELF_FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":100,"text":"[ \"$SELF_FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":101,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":102,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":103,"text":""},{"line":104,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi"},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""},{"line":106,"text":"echo \"$LINE\" | tee \"$HEALTH\" >/dev/null"},{"line":107,"text":"{"},{"line":108,"text":"  echo"},{"line":109,"text":"  echo \"## Result\""},{"line":110,"text":"  echo \"$LINE\""},{"line":111,"text":"} >> \"$TMP\""},{"line":112,"text":"mv \"$TMP\" \"$REPORT\""},{"line":113,"text":"cat \"$HEALTH\""}]},{"bytes":1433,"line_count":27,"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","priority":92,"sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set -Eeuo pipefail"},{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"},{"line":6,"text":"ORIGINAL_RC=0"},{"line":7,"text":"\"$ORIGINAL\" || ORIGINAL_RC=$?"},{"line":8,"text":"[ -s \"$HEALTH\" ] || exit \"$ORIGINAL_RC\""},{"line":9,"text":"line=\"$(sed -n \"1p\" \"$HEALTH\" 2>/dev/null || true)\""},{"line":10,"text":"field(){ local key=<REDACTED>"},{"line":11,"text":"self_failed=\"$(field SELF_FAILED_UNITS || true)\""},{"line":12,"text":"qga_state=\"$(field QGA || true)\""},{"line":13,"text":"backup_state=\"$(field BACKUP || true)\""},{"line":14,"text":"if [ \"$ORIGINAL_RC\" -eq 0 ] && [ \"$self_failed\" = \"0\" ] && [ \"$qga_state\" = \"active\" ] && [ \"$backup_state\" = \"MAIL_CLOUD_2COPY_VERIFY_OK\" ]; then"},{"line":15,"text":"  normalized=\"$(printf \"%s\\n\" \"$line\" | sed -E \"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\")\""},{"line":16,"text":"  printf \"%s\\n\" \"$normalized\" > \"$HEALTH\""},{"line":17,"text":"  if [ -f \"$STATUS_FILE\" ]; then"},{"line":18,"text":"    status_line=\"$(sed -n \"1p\" \"$STATUS_FILE\" 2>/dev/null || true)\""},{"line":19,"text":"    status_normalized=\"$(printf \"%s\\n\" \"$status_line\" | sed -E \"s/^STATUS=[A-Z]+/STATUS=OK/;s/ BAD=[0-9]+/ BAD=0/\")\""},{"line":20,"text":"    tmp=\"${STATUS_FILE}.tmp.$$\""},{"line":21,"text":"    { printf \"%s\\n\" \"$status_normalized\"; tail -n +2 \"$STATUS_FILE\" 2>/dev/null || true; } > \"$tmp\""},{"line":22,"text":"    chown --reference=\"$STATUS_FILE\" \"$tmp\" 2>/dev/null || true"},{"line":23,"text":"    chmod --reference=\"$STATUS_FILE\" \"$tmp\" 2>/dev/null || true"},{"line":24,"text":"    mv \"$tmp\" \"$STATUS_FILE\""},{"line":25,"text":"  fi"},{"line":26,"text":"fi"},{"line":27,"text":"exit \"$ORIGINAL_RC\""}]},{"bytes":848,"line_count":17,"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","priority":92,"sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set +e"},{"line":3,"text":"TS=$(date -u +%Y-%m-%dT%H:%M:%SZ)"},{"line":4,"text":"BAD=0"},{"line":5,"text":"HOST=$(hostname 2>/dev/null || echo unknown)"},{"line":6,"text":"IP_OK=0"},{"line":7,"text":"ip -4 -br a | grep -q \"[PRIVATE_IP]\" && IP_OK=1 || BAD=$((BAD+1))"},{"line":8,"text":"DNS_OK=0"},{"line":9,"text":"getent hosts pve01 >/dev/null 2>&1 || getent hosts gram1.ru >/dev/null 2>&1"},{"line":10,"text":"[ \"$?\" = 0 ] && DNS_OK=1 || BAD=$((BAD+1))"},{"line":11,"text":"FAILED=$(systemctl --failed --no-legend 2>/dev/null | wc -l)"},{"line":12,"text":"[ \"$FAILED\" = 0 ] || BAD=$((BAD+1))"},{"line":13,"text":"QGA=$(systemctl is-active qemu-guest-agent 2>/dev/null || echo inactive)"},{"line":14,"text":"[ \"$QGA\" = active ] || BAD=$((BAD+1))"},{"line":15,"text":"if [ \"$BAD\" = 0 ]; then STATUS=OK; else STATUS=WARN; fi"},{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}]},{"bytes":2541,"line_count":105,"mode":"0o750","path":"/usr/local/sbin/homelab-stage4i-verify","priority":92,"sha256":"bb397f1779ea88bb2646d7e73d3193c04e54c1a83ef38b13a89133ef722f6c6d","source":[{"line":1,"text":"#!/usr/bin/env bash"},{"line":2,"text":"set -euo pipefail"},{"line":3,"text":"export LC_ALL=C"},{"line":4,"text":""},{"line":5,"text":"MARKER=\"CR0019_STAGE4I_IMMUTABLE_V1\""},{"line":6,"text":"STATE_DIR=\"/var/lib/homelab-cluster-admin/stage4i-verification\""},{"line":7,"text":"REPORT=\"$STATE_DIR/latest.env\""},{"line":8,"text":"TMP=\"$(mktemp)\""},{"line":9,"text":""},{"line":10,"text":"cleanup() {"},{"line":11,"text":"    rm -f \"$TMP\""},{"line":12,"text":"}"},{"line":13,"text":"trap cleanup EXIT"},{"line":14,"text":""},{"line":15,"text":"mapfile -t ACTIVATION_FILES < <("},{"line":16,"text":"    find \\"},{"line":17,"text":"        /var/lib/homelab-cluster-admin \\"},{"line":18,"text":"        /opt \\"},{"line":19,"text":"        /srv \\"},{"line":20,"text":"        /usr/local/share \\"},{"line":21,"text":"        -type f \\"},{"line":22,"text":"        -path '*/.stage4i-control/activation.env' \\"},{"line":23,"text":"        -print 2>/dev/null |"},{"line":24,"text":"    sort -u"},{"line":25,"text":")"},{"line":26,"text":""},{"line":27,"text":"if [ \"${#ACTIVATION_FILES[@]}\" -ne 1 ]; then"},{"line":28,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=FAILED\""},{"line":29,"text":"    echo \"REASON=ACTIVATION_FILE_COUNT_${#ACTIVATION_FILES[@]}\""},{"line":30,"text":"    exit 50"},{"line":31,"text":"fi"},{"line":32,"text":""},{"line":33,"text":"ACTIVATION_FILE=\"${ACTIVATION_FILES[0]}\""},{"line":34,"text":"CONTROL_DIR=\"$(dirname \"$ACTIVATION_FILE\")\""},{"line":35,"text":"ACTIVE_LINK=\"$CONTROL_DIR/active\""},{"line":36,"text":"ROLLBACK_LINK=\"$CONTROL_DIR/rollback\""},{"line":37,"text":""},{"line":38,"text":"# shellcheck disable=SC1090"},{"line":39,"text":"source \"$ACTIVATION_FILE\""},{"line":40,"text":""},{"line":41,"text":"test \"${STAGE4I_ACTIVATION_STATUS:-}\" = \"ACTIVE\""},{"line":42,"text":"test -L \"$ACTIVE_LINK\""},{"line":43,"text":"test -L \"$ROLLBACK_LINK\""},{"line":44,"text":""},{"line":45,"text":"ACTIVE_RESOLVED=\"$(readlink -f \"$ACTIVE_LINK\")\""},{"line":46,"text":"ROLLBACK_RESOLVED=\"$(readlink -f \"$ROLLBACK_LINK\")\""},{"line":47,"text":""},{"line":48,"text":"test \"$ACTIVE_RESOLVED\" = \"${ACTIVE_PATH:-}\""},{"line":49,"text":"test \"$ROLLBACK_RESOLVED\" = \"${ROLLBACK_PATH:-}\""},{"line":50,"text":"test -d \"$ACTIVE_RESOLVED\""},{"line":51,"text":"test -d \"$ROLLBACK_RESOLVED\""},{"line":52,"text":""},{"line":53,"text":"META=\"$ACTIVE_RESOLVED/IMMUTABLE.env\""},{"line":54,"text":"MANIFEST=\"$ACTIVE_RESOLVED/MANIFEST.sha256\""},{"line":55,"text":""},{"line":56,"text":"test -f \"$META\""},{"line":57,"text":"test -f \"$MANIFEST\""},{"line":58,"text":"grep -q \"^BUILD_MARKER=$MARKER$\" \"$META\""},{"line":59,"text":"grep -q '^TASK_STAGE=4I$' \"$META\""},{"line":60,"text":"grep -q '^TASK_STATUS=IMMUTABLE$' \"$META\""},{"line":61,"text":""},{"line":62,"text":"("},{"line":63,"text":"    cd \"$ACTIVE_RESOLVED\""},{"line":64,"text":"    sha256sum -c MANIFEST.sha256 >/dev/null"},{"line":65,"text":")"},{"line":66,"text":""},{"line":67,"text":"if find \"$ACTIVE_RESOLVED\" \\"},{"line":68,"text":"    \\( ! -user root -o ! -group root \\) \\"},{"line":69,"text":"    -print -quit |"},{"line":70,"text":"    grep -q ."},{"line":71,"text":"then"},{"line":72,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=FAILED\""},{"line":73,"text":"    echo \"REASON=INVALID_OWNER\""},{"line":74,"text":"    exit 51"},{"line":75,"text":"fi"},{"line":76,"text":""},{"line":77,"text":"if find \"$ACTIVE_RESOLVED\" \\"},{"line":78,"text":"    -perm /0222 \\"},{"line":79,"text":"    -print -quit |"},{"line":80,"text":"    grep -q ."},{"line":81,"text":"then"},{"line":82,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=FAILED\""},{"line":83,"text":"    echo \"REASON=WRITABLE_IMMUTABLE_CONTENT\""},{"line":84,"text":"    exit 52"},{"line":85,"text":"fi"},{"line":86,"text":""},{"line":87,"text":"install -d -m 0750 -o root -g root \"$STATE_DIR\""},{"line":88,"text":""},{"line":89,"text":"{"},{"line":90,"text":"    echo \"STAGE4I_VERIFICATION_STATUS=SUCCESS\""},{"line":91,"text":"    echo \"VERIFIED_AT_UTC=$(date -u +%Y-%m-%dT%H:%M:%SZ)\""},{"line":92,"text":"    echo \"ACTIVE_VERSION=${ACTIVE_VERSION:-UNKNOWN}\""},{"line":93,"text":"    echo \"ACTIVE_PATH=$ACTIVE_RESOLVED\""},{"line":94,"text":"    echo \"ROLLBACK_VERSION=${ROLLBACK_VERSION:-UNKNOWN}\""},{"line":95,"text":"    echo \"ROLLBACK_PATH=$ROLLBACK_RESOLVED\""},{"line":96,"text":"    echo \"MANIFEST_SHA256=$(sha256sum \"$MANIFEST\" | awk '{print $1}')\""},{"line":97,"text":"    echo \"VERIFIED_ON_HOST=$(hostname -f 2>/dev/null || hostname)\""},{"line":98,"text":"} >\"$TMP\""},{"line":99,"text":""},{"line":100,"text":"chown root:root \"$TMP\""},{"line":101,"text":"chmod 0440 \"$TMP\""},{"line":102,"text":"mv -f \"$TMP\" \"$REPORT\""},{"line":103,"text":"trap - EXIT"},{"line":104,"text":""},{"line":105,"text":"cat \"$REPORT\""}]},{"bytes":616,"line_count":19,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","priority":90,"sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":"Requires=postgresql.service"},{"line":6,"text":""},{"line":7,"text":"[Service]"},{"line":8,"text":"Type=oneshot"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":13,"text":"UMask=0027"},{"line":14,"text":"NoNewPrivileges=true"},{"line":15,"text":"PrivateTmp=true"},{"line":16,"text":"ProtectSystem=strict"},{"line":17,"text":"ProtectHome=true"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}]},{"bytes":231,"line_count":12,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","priority":90,"sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=30s"},{"line":6,"text":"OnUnitActiveSec=60s"},{"line":7,"text":"AccuracySec=10s"},{"line":8,"text":"Persistent=true"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"},{"line":10,"text":""},{"line":11,"text":"[Install]"},{"line":12,"text":"WantedBy=timers.target"}]},{"bytes":803,"line_count":28,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","priority":90,"sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":3,"text":"After=network-online.target postgresql.service"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":"Requires=postgresql.service"},{"line":6,"text":""},{"line":7,"text":"[Service]"},{"line":8,"text":"Type=simple"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":13,"text":"ExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1"},{"line":14,"text":"Restart=on-failure"},{"line":15,"text":"RestartSec=5"},{"line":16,"text":"UMask=0027"},{"line":17,"text":"NoNewPrivileges=true"},{"line":18,"text":"PrivateTmp=true"},{"line":19,"text":"ProtectSystem=strict"},{"line":20,"text":"ProtectHome=true"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"},{"line":23,"text":"RestrictSUIDSGID=true"},{"line":24,"text":"LockPersonality=true"},{"line":25,"text":"RestrictRealtime=true"},{"line":26,"text":""},{"line":27,"text":"[Install]"},{"line":28,"text":"WantedBy=multi-user.target"}]},{"bytes":1190,"line_count":42,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","priority":90,"sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":3,"text":"After=network-online.target"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":""},{"line":6,"text":"[Service]"},{"line":7,"text":"Type=oneshot"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":12,"text":"UMask=0077"},{"line":13,"text":"NoNewPrivileges=yes"},{"line":14,"text":"PrivateTmp=yes"},{"line":15,"text":"PrivateDevices=yes"},{"line":16,"text":"ProtectSystem=strict"},{"line":17,"text":"ProtectHome=yes"},{"line":18,"text":"ProtectKernelTunables=yes"},{"line":19,"text":"ProtectKernelModules=yes"},{"line":20,"text":"ProtectKernelLogs=yes"},{"line":21,"text":"ProtectControlGroups=yes"},{"line":22,"text":"ProtectClock=yes"},{"line":23,"text":"ProtectHostname=yes"},{"line":24,"text":"LockPersonality=yes"},{"line":25,"text":"MemoryDenyWriteExecute=yes"},{"line":26,"text":"RestrictSUIDSGID=yes"},{"line":27,"text":"RestrictRealtime=yes"},{"line":28,"text":"RemoveIPC=yes"},{"line":29,"text":"CapabilityBoundingSet="},{"line":30,"text":"AmbientCapabilities="},{"line":31,"text":"RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6"},{"line":32,"text":"IPAddressDeny=any"},{"line":33,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":34,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":35,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":36,"text":"IPAddressAllow=[PRIVATE_IP]"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":40,"text":"StateDirectoryMode=0750"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"},{"line":42,"text":"TimeoutStartSec=70"}]},{"bytes":223,"line_count":12,"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","priority":90,"sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=2min"},{"line":6,"text":"OnUnitActiveSec=60s"},{"line":7,"text":"RandomizedDelaySec=5s"},{"line":8,"text":"Persistent=true"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"},{"line":10,"text":""},{"line":11,"text":"[Install]"},{"line":12,"text":"WantedBy=timers.target"}]},{"bytes":133,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","priority":90,"sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}]},{"bytes":154,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","priority":90,"sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run homelab cluster-admin full observer"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=5min"},{"line":6,"text":"OnUnitActiveSec=5min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]},{"bytes":131,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","priority":90,"sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}]},{"bytes":149,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","priority":90,"sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run homelab cluster-admin observer"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=3min"},{"line":6,"text":"OnUnitActiveSec=5min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]},{"bytes":127,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","priority":90,"sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}]},{"bytes":151,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","priority":90,"sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Run homelab cluster-admin self check"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=2min"},{"line":6,"text":"OnUnitActiveSec=5min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]},{"bytes":138,"line_count":6,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","priority":90,"sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":3,"text":""},{"line":4,"text":"[Service]"},{"line":5,"text":"Type=oneshot"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}]},{"bytes":154,"line_count":10,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","priority":90,"sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=1min"},{"line":6,"text":"OnUnitActiveSec=1min"},{"line":7,"text":"Persistent=true"},{"line":8,"text":""},{"line":9,"text":"[Install]"},{"line":10,"text":"WantedBy=timers.target"}]},{"bytes":301,"line_count":13,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","priority":90,"sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":3,"text":"After=network-online.target"},{"line":4,"text":"Wants=network-online.target"},{"line":5,"text":""},{"line":6,"text":"[Service]"},{"line":7,"text":"Type=simple"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"},{"line":9,"text":"Restart=always"},{"line":10,"text":"RestartSec=5"},{"line":11,"text":""},{"line":12,"text":"[Install]"},{"line":13,"text":"WantedBy=multi-user.target"}]},{"bytes":465,"line_count":22,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.service","priority":90,"sha256":"9b8986a21c0a3c0f569127fe16ab7a0379e244197015c6a7d1e9033b7b1b3f5b","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Verify active immutable Stage 4I task version"},{"line":3,"text":"After=local-fs.target"},{"line":4,"text":""},{"line":5,"text":"[Service]"},{"line":6,"text":"Type=oneshot"},{"line":7,"text":"User=root"},{"line":8,"text":"Group=root"},{"line":9,"text":"ExecStart=/usr/local/sbin/homelab-stage4i-verify"},{"line":10,"text":"NoNewPrivileges=true"},{"line":11,"text":"PrivateDevices=true"},{"line":12,"text":"ProtectClock=true"},{"line":13,"text":"ProtectControlGroups=true"},{"line":14,"text":"ProtectHome=true"},{"line":15,"text":"ProtectKernelLogs=true"},{"line":16,"text":"ProtectKernelModules=true"},{"line":17,"text":"ProtectKernelTunables=true"},{"line":18,"text":"RestrictNamespaces=true"},{"line":19,"text":"RestrictRealtime=true"},{"line":20,"text":"LockPersonality=true"},{"line":21,"text":"MemoryDenyWriteExecute=true"},{"line":22,"text":"UMask=0027"}]},{"bytes":224,"line_count":12,"mode":"0o644","path":"/etc/systemd/system/homelab-stage4i-verify.timer","priority":90,"sha256":"f1c5ff9de2547ad1d45e4e90c84ac4eaefc83b4599d22ffd3f643ce38d76e396","source":[{"line":1,"text":"[Unit]"},{"line":2,"text":"Description=Regularly verify active immutable Stage 4I task version"},{"line":3,"text":""},{"line":4,"text":"[Timer]"},{"line":5,"text":"OnBootSec=2min"},{"line":6,"text":"OnUnitActiveSec=15min"},{"line":7,"text":"AccuracySec=1min"},{"line":8,"text":"Persistent=true"},{"line":9,"text":"Unit=homelab-stage4i-verify.service"},{"line":10,"text":""},{"line":11,"text":"[Install]"},{"line":12,"text":"WantedBy=timers.target"}]},{"bytes":380,"line_count":1,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","priority":70,"sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e","source":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}]},{"bytes":217,"line_count":1,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","priority":70,"sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053","source":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}]},{"bytes":128,"line_count":1,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","priority":70,"sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac","source":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md"}]},{"bytes":1535,"line_count":1,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","priority":70,"sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b","source":[{"line":1,"text":"STATUS=<REDACTED>"}]},{"bytes":89,"line_count":3,"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","priority":50,"sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f","source":[{"line":1,"text":"[Service]"},{"line":2,"text":"ExecStart="},{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}]},{"bytes":28599,"line_count":484,"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","priority":50,"sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6","source":[{"line":1,"text":"<REDACTED_SECRET_LINE>"},{"line":2,"text":"from collections import defaultdict"},{"line":3,"text":"from contextlib import contextmanager"},{"line":4,"text":"from datetime import datetime"},{"line":5,"text":"from urllib.parse import parse_qs"},{"line":6,"text":"import psycopg2"},{"line":7,"text":"from psycopg2.extras import RealDictCursor"},{"line":8,"text":"from fastapi import FastAPI, HTTPException, Request"},{"line":9,"text":"from fastapi.responses import HTMLResponse, RedirectResponse"},{"line":10,"text":"from starlette.middleware.trustedhost import TrustedHostMiddleware"},{"line":11,"text":""},{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE =<REDACTED>"},{"line":14,"text":"COOKIE =<REDACTED>"},{"line":15,"text":"failures, locks = defaultdict(list), {}"},{"line":16,"text":"app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)"},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":18,"text":""},{"line":19,"text":"def db():"},{"line":20,"text":"    return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":21,"text":""},{"line":22,"text":"def enc(value):"},{"line":23,"text":"    return base64.urlsafe_b64encode(value).rstrip(b\"=\").decode()"},{"line":24,"text":""},{"line":25,"text":"def dec(value):"},{"line":26,"text":"    return base64.urlsafe_b64decode(value + \"=\" * (-len(value) % 4))"},{"line":27,"text":""},{"line":28,"text":"def sign(value):"},{"line":29,"text":"    return enc(hmac.new(KEY, value.encode(), hashlib.sha256).digest())"},{"line":30,"text":""},{"line":31,"text":"def ip(request):"},{"line":32,"text":"    peer = request.client.host if request.client else \"unknown\""},{"line":33,"text":"    if peer in {\"[PRIVATE_IP]\",\"127.0.0.1\",\"::1\"}:"},{"line":34,"text":"        return request.headers.get(\"x-forwarded-for\", peer).split(\",\")[0].strip()"},{"line":35,"text":"    return peer"},{"line":36,"text":""},{"line":37,"text":"def login_token(request): <REDACTED>"},{"line":38,"text":"    raw = f\"{int(time.time())}:{ip(request)}\""},{"line":39,"text":"    return raw + \".\" + sign(raw)"},{"line":40,"text":""},{"line":41,"text":"def valid_login_token(request, token): <REDACTED>"},{"line":42,"text":"    try:"},{"line":43,"text":"        raw, signature =<REDACTED>"},{"line":44,"text":"        stamp, address = raw.split(\":\", 1)"},{"line":45,"text":"        return hmac.compare_digest(signature, sign(raw)) and address == ip(request) and 0 <= time.time() - int(stamp) <= 600"},{"line":46,"text":"    except Exception:"},{"line":47,"text":"        return False"},{"line":48,"text":""},{"line":49,"text":"def session_cookie(request): <REDACTED>"},{"line":50,"text":"    data =<REDACTED>"},{"line":51,"text":"    raw = enc(json.dumps(data,separators=(\",\",\":\")).encode())"},{"line":52,"text":"    return raw + \".\" + sign(raw)"},{"line":53,"text":""},{"line":54,"text":"def session(request):"},{"line":55,"text":"    try:"},{"line":56,"text":"        raw, signature =<REDACTED>"},{"line":57,"text":"        data = json.loads(dec(raw))"},{"line":58,"text":"        expected = hashlib.sha256(request.headers.get(\"user-agent\",\"\").encode()).hexdigest()[:24]"},{"line":59,"text":"        return data if hmac.compare_digest(signature,sign(raw)) and data[\"exp\"] >= time.time() and data[\"ua\"] == expected else None"},{"line":60,"text":"    except Exception:"},{"line":61,"text":"        return None"},{"line":62,"text":""},{"line":63,"text":"def password_ok(password): <REDACTED>"},{"line":64,"text":"    try:"},{"line":65,"text":"        scheme, rounds, salt, expected = open(PASSFILE,encoding=\"utf-8\").read().strip().split(\"$\",3)"},{"line":66,"text":"        actual =<REDACTED>"},{"line":67,"text":"        return scheme == \"pbkdf2_sha256\" and hmac.compare_digest(base64.urlsafe_b64encode(actual).decode(),expected)"},{"line":68,"text":"    except Exception:"},{"line":69,"text":"        return False"},{"line":70,"text":""},{"line":71,"text":"def login_page(request, error=\"\"):"},{"line":72,"text":"    token =<REDACTED>"},{"line":73,"text":"    return f\"\"\"<!doctype html><html lang=<REDACTED>"},{"line":74,"text":""},{"line":75,"text":"def status_kind(value):"},{"line":76,"text":"    text = \"UNKNOWN\" if value is None or str(value).strip() == \"\" else str(value).strip().upper()"},{"line":77,"text":"    if text in {\"OK\", \"ACTIVE\", \"RUNNING\", \"UP\", \"ONLINE\", \"HEALTHY\", \"RECOVERED\", \"CLOSED\", \"SUCCESS\", \"ENABLED\", \"READY\"}:"},{"line":78,"text":"        return \"ok\""},{"line":79,"text":"    if text in {\"WARN\", \"WARNING\", \"PENDING\", \"DEGRADED\", \"ATTENTION\", \"STALE\"}:"},{"line":80,"text":"        return \"warn\""},{"line":81,"text":"    if text in {\"OPEN\", \"ERROR\", \"FAILED\", \"DOWN\", \"OFFLINE\", \"CRITICAL\", \"P0\", \"P1\", \"FIRING\", \"UNHEALTHY\"}:"},{"line":82,"text":"        return \"bad\""},{"line":83,"text":"    return \"unknown\""},{"line":84,"text":""},{"line":85,"text":"def status_badge(value):"},{"line":86,"text":"    text = \"UNKNOWN\" if value is None or str(value).strip() == \"\" else str(value)"},{"line":87,"text":"    kind = status_kind(text)"},{"line":88,"text":"    return \"<span class=\\\"state state-\" + kind + \"\\\"><span class=\\\"state-dot\\\"></span>\" + html.escape(text) + \"</span>\""},{"line":89,"text":""},{"line":90,"text":"def summary_card(title, value, state):"},{"line":91,"text":"    return \"<div class=\\\"card\\\"><div class=\\\"card-health\\\">\" + status_badge(state) + \"</div><b>\" + html.escape(str(value)) + \"</b><span>\" + html.escape(title) + \"</span></div>\""},{"line":92,"text":""},{"line":93,"text":"def aggregate_state(rows, keys=(\"status\",), empty_state=\"WARN\"):"},{"line":94,"text":"    if not rows:"},{"line":95,"text":"        return empty_state"},{"line":96,"text":"    kinds = []"},{"line":97,"text":"    for row in rows:"},{"line":98,"text":"        for key in keys:"},{"line":99,"text":"            if row.get(key) is not None:"},{"line":100,"text":"                kinds.append(status_kind(row.get(key)))"},{"line":101,"text":"    if \"bad\" in kinds:"},{"line":102,"text":"        return \"ERROR\""},{"line":103,"text":"    if \"warn\" in kinds or \"unknown\" in kinds or not kinds:"},{"line":104,"text":"        return \"WARN\""},{"line":105,"text":"    return \"OK\""},{"line":106,"text":""},{"line":107,"text":"def incident_section_state(rows):"},{"line":108,"text":"    if not rows:"},{"line":109,"text":"        return \"OK\""},{"line":110,"text":"    severe = {\"P0\", \"P1\", \"CRITICAL\", \"ERROR\"}"},{"line":111,"text":"    for row in rows:"},{"line":112,"text":"        severity = str(row.get(\"severity\") or \"\").strip().upper()"},{"line":113,"text":"        state = row.get(\"status\")"},{"line":114,"text":"        if severity in severe or status_kind(state) == \"bad\":"},{"line":115,"text":"            return \"ERROR\""},{"line":116,"text":"    return \"WARN\""},{"line":117,"text":""},{"line":118,"text":"# Все текущие и будущие подробные блоки dashboard должны использовать этот компонент."},{"line":119,"text":"def health_section(title, count, state, content, wide=False):"},{"line":120,"text":"    kind = status_kind(state)"},{"line":121,"text":"    css_class = \"panel health-section\" + (\" wide\" if wide else \"\")"},{"line":122,"text":"    return \"<details class=\\\"\" + css_class + \"\\\"><summary><span class=\\\"section-heading\\\"><span class=\\\"state state-\" + kind + \"\\\" aria-label=\\\"\" + html.escape(str(state)) + \"\\\"><span class=\\\"state-dot\\\"></span></span><span class=\\\"section-title\\\">\" + html.escape(str(title)) + \"</span><span class=\\\"section-count\\\">\" + html.escape(str(count)) + \"</span></span><span class=\\\"section-chevron\\\" aria-hidden=\\\"true\\\">⌄</span></summary><div class=\\\"section-body\\\">\" + content + \"</div></details>\""},{"line":123,"text":""},{"line":124,"text":"def table(rows, columns):"},{"line":125,"text":"    if not rows:"},{"line":126,"text":"        return \"<p class=\\\"muted\\\">Данные ещё не собраны.</p>\""},{"line":127,"text":"    head = \"\".join(\"<th>\" + html.escape(str(title)) + \"</th>\" for title, key in columns)"},{"line":128,"text":"    body = []"},{"line":129,"text":"    for row in rows:"},{"line":130,"text":"        cells = []"},{"line":131,"text":"        for title, key in columns:"},{"line":132,"text":"            value = row.get(key)"},{"line":133,"text":"            rendered = status_badge(value) if key in {\"status\", \"severity\"} else html.escape(str(value if value is not None else chr(8212)))"},{"line":134,"text":"            cells.append(\"<td>\" + rendered + \"</td>\")"},{"line":135,"text":"        row_kind = status_kind(row.get(\"status\", row.get(\"severity\", \"UNKNOWN\")))"},{"line":136,"text":"        body.append(\"<tr class=\\\"row-\" + row_kind + \"\\\">\" + \"\".join(cells) + \"</tr>\")"},{"line":137,"text":"    return \"<table><thead><tr>\" + head + \"</tr></thead><tbody>\" + \"\".join(body) + \"</tbody></table>\""},{"line":138,"text":""},{"line":139,"text":"@app.middleware(\"http\")"},{"line":140,"text":"async def headers(request, call_next):"},{"line":141,"text":"    response = await call_next(request)"},{"line":142,"text":"    q = chr(39)"},{"line":143,"text":"    response.headers[\"Content-Security-Policy\"] = f\"default-src {q}self{q};style-src {q}self{q} {q}unsafe-inline{q};frame-ancestors {q}none{q};base-uri {q}none{q};form-action {q}self{q}\""},{"line":144,"text":"    response.headers[\"X-Frame-Options\"] = \"DENY\""},{"line":145,"text":"    response.headers[\"X-Content-Type-Options\"] = \"nosniff\""},{"line":146,"text":"    response.headers[\"Referrer-Policy\"] = \"no-referrer\""},{"line":147,"text":"    response.headers[\"Cache-Control\"] = \"no-store\""},{"line":148,"text":"    return response"},{"line":149,"text":""},{"line":150,"text":"@app.get(\"/health\")"},{"line":151,"text":"def health():"},{"line":152,"text":"    return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":153,"text":""},{"line":154,"text":"@app.get(\"/login\",response_class=HTMLResponse)"},{"line":155,"text":"def get_login(request:Request):"},{"line":156,"text":"    return RedirectResponse(\"/\",303) if session(request) else HTMLResponse(login_page(request))"},{"line":157,"text":""},{"line":158,"text":"@app.post(\"/login\",response_class=HTMLResponse)"},{"line":159,"text":"async def post_login(request:Request):"},{"line":160,"text":"    address, now = ip(request), time.time()"},{"line":161,"text":"    failures[address] = [stamp for stamp in failures[address] if now-stamp < 900]"},{"line":162,"text":"    if locks.get(address,0) > now:"},{"line":163,"text":"        return HTMLResponse(login_page(request,\"Слишком много попыток. Повторите позже.\"),429)"},{"line":164,"text":"    if int(request.headers.get(\"content-length\",\"0\") or 0) > 4096:"},{"line":165,"text":"        raise HTTPException(413)"},{"line":166,"text":"    form = parse_qs((await request.body()).decode(\"utf-8\",\"replace\"))"},{"line":167,"text":"    if not valid_login_token(request,form.get(\"csrf\",[\"\"])[0]) or not password_ok(form.get(\"password\",[\"\"])[0]): <REDACTED>"},{"line":168,"text":"        failures[address].append(now)"},{"line":169,"text":"        if len(failures[address]) >= 5:"},{"line":170,"text":"            locks[address], failures[address] = now+900, []"},{"line":171,"text":"        return HTMLResponse(login_page(request,\"Неверный пароль.\"),401)"},{"line":172,"text":"    failures.pop(address,None); locks.pop(address,None)"},{"line":173,"text":"    response = RedirectResponse(\"/\",303)"},{"line":174,"text":"    response.set_cookie(COOKIE,session_cookie(request),max_age=<REDACTED>"},{"line":175,"text":"    return response"},{"line":176,"text":""},{"line":177,"text":"@app.post(\"/logout\")"},{"line":178,"text":"async def logout(request:Request):"},{"line":179,"text":"    data = session(request)"},{"line":180,"text":"    if not data:"},{"line":181,"text":"        raise HTTPException(401)"},{"line":182,"text":"    form = parse_qs((await request.body()).decode(\"utf-8\",\"replace\"))"},{"line":183,"text":"    if not hmac.compare_digest(form.get(\"csrf\",[\"\"])[0],data[\"csrf\"]):"},{"line":184,"text":"        raise HTTPException(403)"},{"line":185,"text":"    response = RedirectResponse(\"/login\",303)"},{"line":186,"text":"    response.delete_cookie(COOKIE,path=<REDACTED>"},{"line":187,"text":"    return response"},{"line":188,"text":""},{"line":189,"text":"@app.get(\"/api/summary\")"},{"line":190,"text":"def api_summary(request:Request):"},{"line":191,"text":"    if not session(request):"},{"line":192,"text":"        raise HTTPException(401)"},{"line":193,"text":"    with db() as connection, connection.cursor(cursor_factory=RealDictCursor) as cursor:"},{"line":194,"text":"        cursor.execute(\"SELECT (SELECT count(*) FROM nodes) nodes,(SELECT count(*) FROM guests) guests,(SELECT count(*) FROM services) services,(SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)) incidents\",(\"RECOVERED\",\"CLOSED\"))"},{"line":195,"text":"        return dict(cursor.fetchone())"},{"line":196,"text":""},{"line":197,"text":"@app.get(\"/\",response_class=HTMLResponse)"},{"line":198,"text":"def dashboard(request:Request):"},{"line":199,"text":"    data = session(request)"},{"line":200,"text":"    if not data:"},{"line":201,"text":"        return RedirectResponse(\"/login\",303)"},{"line":202,"text":"    with db() as connection, connection.cursor(cursor_factory=RealDictCursor) as cursor:"},{"line":203,"text":"        cursor.execute(\"SELECT (SELECT count(*) FROM nodes) nodes,(SELECT count(*) FROM guests) guests,(SELECT count(*) FROM services) services,(SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)) incidents\",(\"RECOVERED\",\"CLOSED\"))"},{"line":204,"text":"        counts = dict(cursor.fetchone())"},{"line":205,"text":"        cursor.execute(\"SELECT name,role,status,last_seen FROM nodes ORDER BY name\"); nodes=[dict(x) for x in cursor.fetchall()]"},{"line":206,"text":"        cursor.execute(\"SELECT name,scope,host_name,container_name,status,last_seen FROM services ORDER BY name LIMIT 250\"); services=[dict(x) for x in cursor.fetchall()]"},{"line":207,"text":"        cursor.execute(\"SELECT severity,title,status,last_seen FROM incidents WHERE status NOT IN (%s,%s) ORDER BY last_seen DESC LIMIT 100\",(\"RECOVERED\",\"CLOSED\")); incidents=[dict(x) for x in cursor.fetchall()]"},{"line":208,"text":"    cards = \"\".join([summary_card(\"Ноды\", counts[\"nodes\"], \"OK\" if counts[\"nodes\"] > 0 else \"ERROR\"), summary_card(\"VM и CT\", counts[\"guests\"], \"OK\" if counts[\"guests\"] > 0 else \"ERROR\"), summary_card(\"Сервисы\", counts[\"services\"], \"OK\" if counts[\"services\"] > 0 else \"ERROR\"), summary_card(\"Инциденты\", counts[\"incidents\"], \"ERROR\" if counts[\"incidents\"] > 0 else \"OK\")])"},{"line":209,"text":"    csrf = data[\"csrf\"]"},{"line":210,"text":"    nodes_table = table(nodes,[(\"Нода\",\"name\"),(\"Роль\",\"role\"),(\"Статус\",\"status\"),(\"Последняя связь\",\"last_seen\")])"},{"line":211,"text":"    incidents_table = table(incidents,[(\"Severity\",\"severity\"),(\"Инцидент\",\"title\"),(\"Статус\",\"status\"),(\"Обновлён\",\"last_seen\")])"},{"line":212,"text":"    services_table = table(services,[(\"Сервис\",\"name\"),(\"Scope\",\"scope\"),(\"Хост\",\"host_name\"),(\"Контейнер\",\"container_name\"),(\"Статус\",\"status\"),(\"Последняя связь\",\"last_seen\")])"},{"line":213,"text":"    nodes_state = aggregate_state(nodes, (\"status\",), \"ERROR\")"},{"line":214,"text":"    incidents_state = incident_section_state(incidents)"},{"line":215,"text":"    services_state = aggregate_state(services, (\"status\",), \"ERROR\")"},{"line":216,"text":"    nodes_html = health_section(\"Ноды\", len(nodes), nodes_state, nodes_table)"},{"line":217,"text":"    incidents_html = health_section(\"Активные инциденты\", len(incidents), incidents_state, incidents_table)"},{"line":218,"text":"    services_html = health_section(\"Сервисы\", len(services), services_state, services_table, wide=True)"},{"line":219,"text":"    engine_badge = status_badge(\"OK\")"},{"line":220,"text":"    page = f\"\"\"<!doctype html><html lang=\"ru\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta http-equiv=\"refresh\" content=\"30\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\"post\" action=\"/logout\"><input type=\"hidden\" name=\"csrf\" value=\"{csrf}\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\"\"\""},{"line":221,"text":"    return HTMLResponse(page)"},{"line":222,"text":""},{"line":223,"text":"from fastapi.responses import PlainTextResponse"},{"line":224,"text":""},{"line":225,"text":"@app.get(\"/metrics\", response_class=PlainTextResponse)"},{"line":226,"text":"def metrics():"},{"line":227,"text":"    with db() as connection, connection.cursor() as cursor:"},{"line":228,"text":"        cursor.execute(\"SELECT count(*) FROM nodes\")"},{"line":229,"text":"        nodes = cursor.fetchone()[0]"},{"line":230,"text":"        cursor.execute(\"SELECT count(*) FROM guests\")"},{"line":231,"text":"        guests = cursor.fetchone()[0]"},{"line":232,"text":"        cursor.execute(\"SELECT count(*) FROM services\")"},{"line":233,"text":"        services = cursor.fetchone()[0]"},{"line":234,"text":"        cursor.execute(\"SELECT count(*) FROM observations\")"},{"line":235,"text":"        observations = cursor.fetchone()[0]"},{"line":236,"text":"        cursor.execute(\"SELECT count(*) FROM incidents WHERE status NOT IN (%s,%s)\", (\"RECOVERED\",\"CLOSED\"))"},{"line":237,"text":"        incidents = cursor.fetchone()[0]"},{"line":238,"text":"        cursor.execute(\"SELECT COALESCE(EXTRACT(EPOCH FROM (now()-max(finished_at))),999999) FROM collector_runs WHERE status=%s\", (\"OK\",))"},{"line":239,"text":"        collector_age = float(cursor.fetchone()[0])"},{"line":240,"text":"        cursor.execute(\"SELECT severity,count(*) FROM incidents WHERE status NOT IN (%s,%s) GROUP BY severity ORDER BY severity\", (\"RECOVERED\",\"CLOSED\"))"},{"line":241,"text":"        severity_rows = cursor.fetchall()"},{"line":242,"text":"    lines = ["},{"line":243,"text":"        \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":"        \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":"        \"cluster_admin_up 1\","},{"line":246,"text":"        \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":"        \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":"        \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":"        \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":"        \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":"        \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":"        \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":"        \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":"        \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":"        \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":"        \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":"        \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":"        \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":"        \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":"        \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":"        \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":"        \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":"        \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":264,"text":"    ]"},{"line":265,"text":"    for severity, count in severity_rows:"},{"line":266,"text":"        clean = str(severity).replace(chr(34), \"\")"},{"line":267,"text":"        lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"},{"line":268,"text":"    return PlainTextResponse(\"\\n\".join(lines) + \"\\n\", media_type=\"text/plain; version=0.0.4\")"},{"line":269,"text":""},{"line":270,"text":"@contextmanager"},{"line":271,"text":"def api_v1_read_cursor():"},{"line":272,"text":"    connection = db()"},{"line":273,"text":"    connection.autocommit = True"},{"line":274,"text":"    cursor = connection.cursor(cursor_factory=RealDictCursor)"},{"line":275,"text":"    try:"},{"line":276,"text":"        cursor.execute(\"BEGIN READ ONLY\")"},{"line":277,"text":"        cursor.execute(\"SET LOCAL statement_timeout TO 5000\")"},{"line":278,"text":"        yield cursor"},{"line":279,"text":"    finally:"},{"line":280,"text":"        try:"},{"line":281,"text":"            cursor.execute(\"ROLLBACK\")"},{"line":282,"text":"        except Exception:"},{"line":283,"text":"            pass"},{"line":284,"text":"        cursor.close()"},{"line":285,"text":"        connection.close()"},{"line":286,"text":""},{"line":287,"text":""},{"line":288,"text":"def api_v1_require_session(request: Request):"},{"line":289,"text":"    data = session(request)"},{"line":290,"text":"    if not data:"},{"line":291,"text":"        raise HTTPException(401)"},{"line":292,"text":"    return data"},{"line":293,"text":""},{"line":294,"text":""},{"line":295,"text":"def api_v1_limit(value):"},{"line":296,"text":"    if value < 1 or value > 500:"},{"line":297,"text":"        raise HTTPException(422, detail=\"limit must be between 1 and 500\")"},{"line":298,"text":"    return value"},{"line":299,"text":""},{"line":300,"text":""},{"line":301,"text":"def api_v1_updated_since(value):"},{"line":302,"text":"    if not value:"},{"line":303,"text":"        return None"},{"line":304,"text":"    try:"},{"line":305,"text":"        return datetime.fromisoformat(value.replace(\"Z\", \"+00:00\"))"},{"line":306,"text":"    except Exception:"},{"line":307,"text":"        raise HTTPException(422, detail=\"updated_since must be ISO 8601\")"},{"line":308,"text":""},{"line":309,"text":""},{"line":310,"text":"def api_v1_encode_id_cursor(value):"},{"line":311,"text":"    return enc(str(int(value)).encode())"},{"line":312,"text":""},{"line":313,"text":""},{"line":314,"text":"def api_v1_decode_id_cursor(value):"},{"line":315,"text":"    if not value:"},{"line":316,"text":"        return None"},{"line":317,"text":"    try:"},{"line":318,"text":"        result = int(dec(value).decode())"},{"line":319,"text":"        if result < 0:"},{"line":320,"text":"            raise ValueError()"},{"line":321,"text":"        return result"},{"line":322,"text":"    except Exception:"},{"line":323,"text":"        raise HTTPException(422, detail=\"invalid cursor\")"},{"line":324,"text":""},{"line":325,"text":""},{"line":326,"text":"@app.get(\"/api/v1/entities\")"},{"line":327,"text":"def api_v1_entities(request: Request, limit: int = 100, cursor: str = \"\", entity_type: str = \"\", lifecycle_status: str = \"\", source_of_truth: str = \"\", updated_since: str = \"\", search: str = \"\"):"},{"line":328,"text":"    api_v1_require_session(request)"},{"line":329,"text":"    limit = api_v1_limit(limit)"},{"line":330,"text":"    clauses = []"},{"line":331,"text":"    params = []"},{"line":332,"text":"    if cursor:"},{"line":333,"text":"        clauses.append(\"e.entity_key > %s\")"},{"line":334,"text":"        params.append(cursor)"},{"line":335,"text":"    if entity_type:"},{"line":336,"text":"        clauses.append(\"e.entity_type = %s\")"},{"line":337,"text":"        params.append(entity_type)"},{"line":338,"text":"    if lifecycle_status:"},{"line":339,"text":"        clauses.append(\"e.lifecycle_status = %s\")"},{"line":340,"text":"        params.append(lifecycle_status)"},{"line":341,"text":"    if source_of_truth:"},{"line":342,"text":"        clauses.append(\"e.source_of_truth = %s\")"},{"line":343,"text":"        params.append(source_of_truth)"},{"line":344,"text":"    since = api_v1_updated_since(updated_since)"},{"line":345,"text":"    if since is not None:"},{"line":346,"text":"        clauses.append(\"e.updated_at >= %s\")"},{"line":347,"text":"        params.append(since)"},{"line":348,"text":"    if search:"},{"line":349,"text":"        clauses.append(\"(e.entity_key ILIKE %s OR e.display_name ILIKE %s)\")"},{"line":350,"text":"        params.extend([\"%\" + search + \"%\", \"%\" + search + \"%\"])"},{"line":351,"text":"    sql = \"SELECT e.entity_key,e.entity_type,e.display_name,e.lifecycle_status,e.source_of_truth,e.created_at,e.updated_at FROM entities e\""},{"line":352,"text":"    if clauses:"},{"line":353,"text":"        sql += \" WHERE \" + \" AND \".join(clauses)"},{"line":354,"text":"    sql += \" ORDER BY e.entity_key LIMIT %s\""},{"line":355,"text":"    params.append(limit + 1)"},{"line":356,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":357,"text":"        db_cursor.execute(sql, params)"},{"line":358,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":359,"text":"    page = rows[:limit]"},{"line":360,"text":"    next_cursor = page[-1][\"entity_key\"] if len(rows) > limit else None"},{"line":361,"text":"    return {\"items\": page, \"next_cursor\": next_cursor, \"limit\": limit}"},{"line":362,"text":""},{"line":363,"text":""},{"line":364,"text":"@app.get(\"/api/v1/entities/{entity_key:path}/dependencies\")"},{"line":365,"text":"def api_v1_entity_dependencies(request: Request, entity_key: str, limit: int = 100, cursor: str = \"\"):"},{"line":366,"text":"    api_v1_require_session(request)"},{"line":367,"text":"    limit = api_v1_limit(limit)"},{"line":368,"text":"    cursor_id = api_v1_decode_id_cursor(cursor)"},{"line":369,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":370,"text":"        db_cursor.execute(\"SELECT id FROM entities WHERE entity_key=%s\", (entity_key,))"},{"line":371,"text":"        entity = db_cursor.fetchone()"},{"line":372,"text":"        if not entity:"},{"line":373,"text":"            raise HTTPException(404, detail=\"entity not found\")"},{"line":374,"text":"        entity_id = entity[\"id\"]"},{"line":375,"text":"        clauses = [\"(d.parent_entity_id=%s OR d.child_entity_id=%s)\"]"},{"line":376,"text":"        params = [entity_id, entity_id, entity_id]"},{"line":377,"text":"        if cursor_id is not None:"},{"line":378,"text":"            clauses.append(\"d.id > %s\")"},{"line":379,"text":"            params.append(cursor_id)"},{"line":380,"text":"        sql = \"SELECT d.id AS _cursor_id,CASE WHEN d.parent_entity_id=%s THEN 'outgoing' ELSE 'incoming' END AS direction,p.entity_key AS parent_entity_key,c.entity_key AS child_entity_key,d.relation_type,d.source_relation,d.criticality,d.edge_state,d.created_at,d.updated_at FROM entity_dependencies d JOIN entities p ON p.id=d.parent_entity_id JOIN entities c ON c.id=d.child_entity_id WHERE \" + \" AND \".join(clauses) + \" ORDER BY d.id LIMIT %s\""},{"line":381,"text":"        params.append(limit + 1)"},{"line":382,"text":"        db_cursor.execute(sql, params)"},{"line":383,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":384,"text":"    page = rows[:limit]"},{"line":385,"text":"    next_cursor = api_v1_encode_id_cursor(page[-1][\"_cursor_id\"]) if len(rows) > limit else None"},{"line":386,"text":"    for row in page:"},{"line":387,"text":"        row.pop(\"_cursor_id\", None)"},{"line":388,"text":"    return {\"entity_key\": entity_key, \"items\": page, \"next_cursor\": next_cursor, \"limit\": limit}"},{"line":389,"text":""},{"line":390,"text":""},{"line":391,"text":"@app.get(\"/api/v1/entities/{entity_key:path}\")"},{"line":392,"text":"def api_v1_entity_detail(request: Request, entity_key: str):"},{"line":393,"text":"    api_v1_require_session(request)"},{"line":394,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":395,"text":"        db_cursor.execute(\"SELECT id AS _entity_id,entity_key,entity_type,display_name,lifecycle_status,source_of_truth,legacy_table,created_at,updated_at FROM entities WHERE entity_key=%s\", (entity_key,))"},{"line":396,"text":"        row = db_cursor.fetchone()"},{"line":397,"text":"        if not row:"},{"line":398,"text":"            raise HTTPException(404, detail=\"entity not found\")"},{"line":399,"text":"        result = dict(row)"},{"line":400,"text":"        entity_id = result.pop(\"_entity_id\")"},{"line":401,"text":"        db_cursor.execute(\"SELECT alias,alias_type,priority,is_canonical,created_at FROM entity_aliases WHERE entity_id=%s ORDER BY is_canonical DESC,priority,alias,alias_type\", (entity_id,))"},{"line":402,"text":"        result[\"aliases\"] = [dict(alias) for alias in db_cursor.fetchall()]"},{"line":403,"text":"    return result"},{"line":404,"text":""},{"line":405,"text":""},{"line":406,"text":"@app.get(\"/api/v1/dependencies\")"},{"line":407,"text":"def api_v1_dependencies(request: Request, limit: int = 100, cursor: str = \"\", parent_entity_key: str = \"\", child_entity_key: str = \"\", relation_type: str = \"\", source_relation: str = \"\", criticality: str = \"\", edge_state: str = \"\"):"},{"line":408,"text":"    api_v1_require_session(request)"},{"line":409,"text":"    limit = api_v1_limit(limit)"},{"line":410,"text":"    cursor_id = api_v1_decode_id_cursor(cursor)"},{"line":411,"text":"    clauses = []"},{"line":412,"text":"    params = []"},{"line":413,"text":"    if cursor_id is not None:"},{"line":414,"text":"        clauses.append(\"d.id > %s\")"},{"line":415,"text":"        params.append(cursor_id)"},{"line":416,"text":"    filters = [(\"p.entity_key\", parent_entity_key), (\"c.entity_key\", child_entity_key), (\"d.relation_type\", relation_type), (\"d.source_relation\", source_relation), (\"d.criticality\", criticality), (\"d.edge_state\", edge_state)]"},{"line":417,"text":"    for column, value in filters:"},{"line":418,"text":"        if value:"},{"line":419,"text":"            clauses.append(column + \" = %s\")"},{"line":420,"text":"            params.append(value)"},{"line":421,"text":"    sql = \"SELECT d.id AS _cursor_id,p.entity_key AS parent_entity_key,c.entity_key AS child_entity_key,p.entity_type AS parent_entity_type,c.entity_type AS child_entity_type,d.relation_type,d.source_relation,d.criticality,d.edge_state,d.created_at,d.updated_at FROM entity_dependencies d JOIN entities p ON p.id=d.parent_entity_id JOIN entities c ON c.id=d.child_entity_id\""},{"line":422,"text":"    if clauses:"},{"line":423,"text":"        sql += \" WHERE \" + \" AND \".join(clauses)"},{"line":424,"text":"    sql += \" ORDER BY d.id LIMIT %s\""},{"line":425,"text":"    params.append(limit + 1)"},{"line":426,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":427,"text":"        db_cursor.execute(sql, params)"},{"line":428,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":429,"text":"    page = rows[:limit]"},{"line":430,"text":"    next_cursor = api_v1_encode_id_cursor(page[-1][\"_cursor_id\"]) if len(rows) > limit else None"},{"line":431,"text":"    for row in page:"},{"line":432,"text":"        row.pop(\"_cursor_id\", None)"},{"line":433,"text":"    return {\"items\": page, \"next_cursor\": next_cursor, \"limit\": limit}"},{"line":434,"text":""},{"line":435,"text":""},{"line":436,"text":"@app.get(\"/api/v1/sources\")"},{"line":437,"text":"def api_v1_sources(request: Request):"},{"line":438,"text":"    api_v1_require_session(request)"},{"line":439,"text":"    sql = \"SELECT s.source_key,s.source_type,s.enabled,si.instance_key,si.status AS instance_status,si.last_seen,se.expectation_type,se.interval_seconds,se.grace_seconds,se.severity,s.created_at,s.updated_at FROM sources s LEFT JOIN source_instances si ON si.source_id=s.id LEFT JOIN source_expectations se ON se.source_instance_id=si.id ORDER BY s.source_key,si.instance_key,se.expectation_type\""},{"line":440,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":441,"text":"        db_cursor.execute(sql)"},{"line":442,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":443,"text":"    return {\"items\": rows}"},{"line":444,"text":""},{"line":445,"text":""},{"line":446,"text":"@app.get(\"/api/v1/policies\")"},{"line":447,"text":"def api_v1_policies(request: Request):"},{"line":448,"text":"    api_v1_require_session(request)"},{"line":449,"text":"    sql = \"SELECT pr.policy_key,pr.scope,pr.enabled,pr.active_version,pv.document->>'mode' AS mode,pv.document->>'automatic_actions' AS automatic_actions,pv.document->>'human_approval_required' AS human_approval_required,pr.created_at,pr.updated_at FROM policy_registry pr LEFT JOIN policy_versions pv ON pv.policy_id=pr.id AND pv.version=pr.active_version ORDER BY pr.policy_key,pr.scope\""},{"line":450,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":451,"text":"        db_cursor.execute(sql)"},{"line":452,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":453,"text":"    return {\"items\": rows}"},{"line":454,"text":""},{"line":455,"text":""},{"line":456,"text":"@app.get(\"/api/v1/collector-runs\")"},{"line":457,"text":"def api_v1_collector_runs(request: Request, limit: int = 100, cursor: str = \"\", collector: str = \"\", status: str = \"\"):"},{"line":458,"text":"    api_v1_require_session(request)"},{"line":459,"text":"    limit = api_v1_limit(limit)"},{"line":460,"text":"    cursor_id = api_v1_decode_id_cursor(cursor)"},{"line":461,"text":"    clauses = []"},{"line":462,"text":"    params = []"},{"line":463,"text":"    if cursor_id is not None:"},{"line":464,"text":"        clauses.append(\"cr.id < %s\")"},{"line":465,"text":"        params.append(cursor_id)"},{"line":466,"text":"    if collector:"},{"line":467,"text":"        clauses.append(\"cr.collector = %s\")"},{"line":468,"text":"        params.append(collector)"},{"line":469,"text":"    if status:"},{"line":470,"text":"        clauses.append(\"cr.status = %s\")"},{"line":471,"text":"        params.append(status)"},{"line":472,"text":"    sql = \"SELECT cr.id AS _cursor_id,cr.collector,cr.status,cr.trigger_kind,cr.schema_version,cr.records_seen,cr.warning_count,cr.run_key,si.instance_key AS source_instance_key,cr.started_at,cr.finished_at FROM collector_runs cr LEFT JOIN source_instances si ON si.id=cr.source_instance_id\""},{"line":473,"text":"    if clauses:"},{"line":474,"text":"        sql += \" WHERE \" + \" AND \".join(clauses)"},{"line":475,"text":"    sql += \" ORDER BY cr.id DESC LIMIT %s\""},{"line":476,"text":"    params.append(limit + 1)"},{"line":477,"text":"    with api_v1_read_cursor() as db_cursor:"},{"line":478,"text":"        db_cursor.execute(sql, params)"},{"line":479,"text":"        rows = [dict(row) for row in db_cursor.fetchall()]"},{"line":480,"text":"    page = rows[:limit]"},{"line":481,"text":"    next_cursor = api_v1_encode_id_cursor(page[-1][\"_cursor_id\"]) if len(rows) > limit else None"},{"line":482,"text":"    for row in page:"},{"line":483,"text":"        row.pop(\"_cursor_id\", None)"},{"line":484,"text":"    return {\"items\": page, \"next_cursor\": next_cursor, \"limit\": limit}"}]},{"bytes":10640,"line_count":113,"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","priority":50,"sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe","source":[{"line":1,"text":"#!/usr/bin/env python3"},{"line":2,"text":"import datetime"},{"line":3,"text":"import json"},{"line":4,"text":"import re"},{"line":5,"text":"import urllib.parse"},{"line":6,"text":"import urllib.request"},{"line":7,"text":"import psycopg2"},{"line":8,"text":"from psycopg2.extras import Json"},{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":11,"text":"PROMETHEUS = \"http://[PRIVATE_IP]:9090\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":13,"text":"def prometheus(expression):"},{"line":14,"text":"    try:"},{"line":15,"text":"        url = PROMETHEUS + \"/api/v1/query?\" + urllib.parse.urlencode({\"query\":expression})"},{"line":16,"text":"        with urllib.request.urlopen(url, timeout=12) as response:"},{"line":17,"text":"            body = json.loads(response.read().decode(\"utf-8\"))"},{"line":18,"text":"        return body.get(\"data\", {}).get(\"result\", []) if body.get(\"status\") == \"success\" else []"},{"line":19,"text":"    except Exception:"},{"line":20,"text":"        return []"},{"line":21,"text":"def numeric(value):"},{"line":22,"text":"    try:"},{"line":23,"text":"        return float(value)"},{"line":24,"text":"    except Exception:"},{"line":25,"text":"        return None"},{"line":26,"text":"def slug(value):"},{"line":27,"text":"    return re.sub(r\"[^a-z0-9]+\", \"-\", str(value).lower()).strip(\"-\")"},{"line":28,"text":"def entity(instance):"},{"line":29,"text":"    address = instance.rsplit(\":\", 1)[0]"},{"line":30,"text":"    return HOSTS.get(address, address)"},{"line":31,"text":"def open_incident(cursor, fingerprint, severity, title, summary, cause, metadata):"},{"line":32,"text":"    cursor.execute(\"INSERT INTO incidents(fingerprint,status,severity,title,summary,root_cause,first_seen,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,now(),now(),%s) ON CONFLICT(fingerprint) DO UPDATE SET status=EXCLUDED.status,severity=EXCLUDED.severity,title=EXCLUDED.title,summary=EXCLUDED.summary,root_cause=EXCLUDED.root_cause,last_seen=now(),recovered_at=NULL,metadata=EXCLUDED.metadata RETURNING id\",(fingerprint,\"OPEN\",severity,title,summary,cause,Json(metadata)))"},{"line":33,"text":"    incident_id = cursor.fetchone()[0]"},{"line":34,"text":"    cursor.execute(\"INSERT INTO incident_events(incident_id,event_type,message,metadata) SELECT %s,%s,%s,%s WHERE NOT EXISTS(SELECT 1 FROM incident_events WHERE incident_id=%s AND event_type=%s AND created_at>now()-make_interval(mins=>10))\",(incident_id,\"OBSERVED\",summary,Json(metadata),incident_id,\"OBSERVED\"))"},{"line":35,"text":"def main():"},{"line":36,"text":"    data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":"    connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":38,"text":"    active = set()"},{"line":39,"text":"    metric_count = 0"},{"line":40,"text":"    with connection.cursor() as cursor:"},{"line":41,"text":"        cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":42,"text":"        run_id = cursor.fetchone()[0]"},{"line":43,"text":"        for item in data.get(\"nodes\", []):"},{"line":44,"text":"            cursor.execute(\"INSERT INTO nodes(name,address,role,status,last_seen,metadata) VALUES(%s,%s,%s,%s,now(),%s) ON CONFLICT(name) DO UPDATE SET address=EXCLUDED.address,role=EXCLUDED.role,status=EXCLUDED.status,last_seen=now(),metadata=EXCLUDED.metadata\",(item.get(\"name\"),item.get(\"address\"),item.get(\"role\"),item.get(\"status\"),Json(item)))"},{"line":45,"text":"        for item in data.get(\"guests\", []):"},{"line":46,"text":"            cursor.execute(\"INSERT INTO guests(guest_key,node_name,guest_type,name,status,cpu_usage,memory_used_bytes,memory_total_bytes,disk_used_bytes,disk_total_bytes,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,now(),%s) ON CONFLICT(guest_key) DO UPDATE SET node_name=EXCLUDED.node_name,guest_type=EXCLUDED.guest_type,name=EXCLUDED.name,status=EXCLUDED.status,cpu_usage=EXCLUDED.cpu_usage,memory_used_bytes=EXCLUDED.memory_used_bytes,memory_total_bytes=EXCLUDED.memory_total_bytes,disk_used_bytes=EXCLUDED.disk_used_bytes,disk_total_bytes=EXCLUDED.disk_total_bytes,last_seen=now(),metadata=EXCLUDED.metadata\",(item.get(\"guest_key\"),item.get(\"node_name\"),item.get(\"guest_type\"),item.get(\"name\"),item.get(\"status\"),item.get(\"cpu_usage\"),item.get(\"memory_used_bytes\"),item.get(\"memory_total_bytes\"),item.get(\"disk_used_bytes\"),item.get(\"disk_total_bytes\"),Json(item)))"},{"line":47,"text":"        for item in data.get(\"services\", []):"},{"line":48,"text":"            service_key = item.get(\"service_key\") or slug(item.get(\"name\"))"},{"line":49,"text":"            cursor.execute(\"INSERT INTO services(service_key,name,scope,host_name,container_name,public_url,internal_url,status,last_seen,metadata) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,now(),%s) ON CONFLICT(service_key) DO UPDATE SET name=EXCLUDED.name,scope=EXCLUDED.scope,host_name=EXCLUDED.host_name,container_name=EXCLUDED.container_name,public_url=EXCLUDED.public_url,internal_url=EXCLUDED.internal_url,status=EXCLUDED.status,last_seen=now(),metadata=EXCLUDED.metadata\",(service_key,item.get(\"name\"),item.get(\"scope\"),item.get(\"host_name\"),item.get(\"container_name\"),item.get(\"public_url\"),item.get(\"internal_url\"),item.get(\"status\"),Json(item)))"},{"line":50,"text":"            if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":51,"text":"                fingerprint = \"service:\" + service_key"},{"line":52,"text":"                active.add(fingerprint)"},{"line":53,"text":"                open_incident(cursor,fingerprint,\"P1\",\"Service requires attention: \" + str(item.get(\"name\")),\"Readiness status is \" + str(item.get(\"status\")),\"service-readiness\",item)"},{"line":54,"text":"        cursor.execute(\"DELETE FROM dependencies\")"},{"line":55,"text":"        for item in data.get(\"dependencies\", []):"},{"line":56,"text":"            cursor.execute(\"INSERT INTO dependencies(parent_key,child_key,relation,metadata) VALUES(%s,%s,%s,%s) ON CONFLICT(parent_key,child_key,relation) DO UPDATE SET metadata=EXCLUDED.metadata\",(item.get(\"parent_key\"),item.get(\"child_key\"),item.get(\"relation\"),Json(item)))"},{"line":57,"text":"        cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":58,"text":"        expressions = {\"up\":\"up\",\"cpu_used_pct\":\"100-(avg by(instance)(rate(node_cpu_seconds_total{mode=\\\"idle\\\"}[5m]))*100)\",\"memory_used_pct\":\"100*(1-(node_memory_MemAvailable_bytes/node_memory_MemTotal_bytes))\",\"filesystem_used_pct\":\"100*(1-(node_filesystem_avail_bytes{fstype!~\\\"tmpfs|overlay|squashfs|nsfs\\\"}/node_filesystem_size_bytes{fstype!~\\\"tmpfs|overlay|squashfs|nsfs\\\"}))\",\"load1\":\"node_load1\"}"},{"line":59,"text":"        for metric, expression in expressions.items():"},{"line":60,"text":"            for result in prometheus(expression):"},{"line":61,"text":"                labels = result.get(\"metric\", {})"},{"line":62,"text":"                instance = labels.get(\"instance\", \"unknown\")"},{"line":63,"text":"                entity_key = entity(instance)"},{"line":64,"text":"                value = numeric(result.get(\"value\", [None, None])[1])"},{"line":65,"text":"                if value is None:"},{"line":66,"text":"                    continue"},{"line":67,"text":"                status = \"OK\""},{"line":68,"text":"                severity = \"P2\""},{"line":69,"text":"                if metric == \"up\" and value < 1:"},{"line":70,"text":"                    status, severity = \"BAD\", \"P1\""},{"line":71,"text":"                elif metric == \"cpu_used_pct\" and value >= 97:"},{"line":72,"text":"                    status, severity = \"BAD\", \"P1\""},{"line":73,"text":"                elif metric == \"cpu_used_pct\" and value >= 90:"},{"line":74,"text":"                    status = \"WARN\""},{"line":75,"text":"                elif metric == \"memory_used_pct\" and value >= 95:"},{"line":76,"text":"                    status, severity = \"BAD\", \"P1\""},{"line":77,"text":"                elif metric == \"memory_used_pct\" and value >= 88:"},{"line":78,"text":"                    status = \"WARN\""},{"line":79,"text":"                elif metric == \"filesystem_used_pct\" and value >= 95:"},{"line":80,"text":"                    status, severity = \"BAD\", \"P1\""},{"line":81,"text":"                elif metric == \"filesystem_used_pct\" and value >= 85:"},{"line":82,"text":"                    status = \"WARN\""},{"line":83,"text":"                unit = \"percent\" if metric.endswith(\"pct\") else \"state\" if metric == \"up\" else \"load\""},{"line":84,"text":"                metadata = {\"instance\":instance,\"job\":labels.get(\"job\"),\"mountpoint\":labels.get(\"mountpoint\"),\"device\":labels.get(\"device\")}"},{"line":85,"text":"                cursor.execute(\"INSERT INTO observations(entity_type,entity_key,metric,value,unit,status,observed_at,metadata) VALUES(%s,%s,%s,%s,%s,%s,now(),%s)\",(\"target\",entity_key,metric,value,unit,status,Json(metadata)))"},{"line":86,"text":"                metric_count += 1"},{"line":87,"text":"                if status != \"OK\":"},{"line":88,"text":"                    cursor.execute(\"SELECT count(*) FROM observations WHERE entity_type=%s AND entity_key=%s AND metric=%s AND status=ANY(%s) AND observed_at>now()-make_interval(mins=>10)\",(\"target\",entity_key,metric,[\"WARN\",\"BAD\"]))"},{"line":89,"text":"                    if cursor.fetchone()[0] >= 2:"},{"line":90,"text":"                        fingerprint = \"metric:\" + slug(entity_key) + \":\" + metric + \":\" + slug(labels.get(\"mountpoint\", \"root\"))"},{"line":91,"text":"                        active.add(fingerprint)"},{"line":92,"text":"                        open_incident(cursor,fingerprint,severity,entity_key + \" - \" + metric,metric + \"=\" + format(value, \".2f\") + \" \" + unit,\"target-unavailable\" if metric == \"up\" else \"resource-pressure\",metadata)"},{"line":93,"text":"        for name, state in data.get(\"backups\", {}).items():"},{"line":94,"text":"            if state.get(\"status\") != \"success\" or not state.get(\"local_backup_deleted\", False) or not state.get(\"remote_a_job_path\") or not state.get(\"remote_b_job_path\"):"},{"line":95,"text":"                fingerprint = \"backup:\" + slug(name)"},{"line":96,"text":"                active.add(fingerprint)"},{"line":97,"text":"                open_incident(cursor,fingerprint,\"P1\",\"Backup requires attention: \" + name,\"Backup is not fully verified\",\"backup-verification\",state)"},{"line":98,"text":"        cursor.execute(\"SELECT id,fingerprint FROM incidents WHERE status=ANY(%s)\",([\"OPEN\",\"DETECTED\",\"ACKNOWLEDGED\",\"MONITORING\"],))"},{"line":99,"text":"        for incident_id, fingerprint in cursor.fetchall():"},{"line":100,"text":"            if fingerprint not in active:"},{"line":101,"text":"                cursor.execute(\"UPDATE incidents SET status=%s,recovered_at=now(),last_seen=now() WHERE id=%s\",(\"RECOVERED\",incident_id))"},{"line":102,"text":"                cursor.execute(\"INSERT INTO incident_events(incident_id,event_type,message,metadata) VALUES(%s,%s,%s,%s)\",(incident_id,\"RECOVERED\",\"Condition returned to normal\",Json({})))"},{"line":103,"text":"        cursor.execute(\"DELETE FROM observations WHERE observed_at<now()-make_interval(days=>30)\")"},{"line":104,"text":"        cursor.execute(\"UPDATE collector_runs SET status=%s,finished_at=now(),metadata=%s WHERE id=%s\",(\"OK\",Json({\"nodes\":len(data.get(\"nodes\",[])),\"guests\":len(data.get(\"guests\",[])),\"services\":len(data.get(\"services\",[])),\"metrics\":metric_count}),run_id))"},{"line":105,"text":"        cursor.execute(\"SELECT count(*) FROM incidents WHERE status=ANY(%s)\",([\"OPEN\",\"DETECTED\",\"ACKNOWLEDGED\",\"MONITORING\"],))"},{"line":106,"text":"        open_count = cursor.fetchone()[0]"},{"line":107,"text":"    connection.commit()"},{"line":108,"text":"    line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""},{"line":109,"text":"    with open(HEALTH, \"w\", encoding=\"utf-8\") as stream:"},{"line":110,"text":"        stream.write(line)"},{"line":111,"text":"    print(line.strip())"},{"line":112,"text":"if __name__ == \"__main__\":"},{"line":113,"text":"    main()"}]}],"gate":{"exact_source_count":30,"index_source_count":1,"ready_for_atomic_design":true,"reference_source_count":1,"unit_record_count":16},"hostname":"cluster-admin","index_paths":["/var/www/homelab-cluster-admin/index.html"],"input_counts":{"application_candidates":40,"health_input_files":4,"indexes":1,"reference_files":1,"units":16},"reference_paths":["/opt/homelab-cluster-admin/render-webpanel.sh"],"unit_records":[{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006178 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","Group":"clusteradmin","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:27 UTC] ; stop_time=[Wed 2026-08-05 13:20:28 UTC] ; pid=2006164 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:15 UTC] ; pid=2005776 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005777 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:19:08 UTC] ; stop_time=[Wed 2026-08-05 13:19:08 UTC] ; pid=2005778 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:20:48 UTC] ; stop_time=[Wed 2026-08-05 13:20:48 UTC] ; pid=2006179 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","Group":"","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 13:11:27 UTC] ; stop_time=[Wed 2026-08-05 13:11:27 UTC] ; pid=2004608 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","Group":"root","LoadState":"loaded","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","User":"root","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"cat_stderr":"","show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}]},"rollback_restored":false,"rollback_started":false,"schema":18,"status":"OK","transaction_gate":{"index_source_found":true,"next_required_action":"build corrected atomic local+remote transaction","ready_for_atomic_transaction":true,"reference_source_found":true,"remote_probe_root":true,"unit_records_found":true},"transport":{"attempts":[{"privilege":"sudo-root","rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":139487,"stdout_sha256":"a710627984d38927f5e0fdde30c8beab966ad52581868b9928ea458dc6902bf3"}],"privilege":"sudo-root","success":true},"v15_candidate_manifest":{"application_candidates":[{"bytes":133,"matches":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},{"bytes":149,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},{"bytes":154,"matches":[{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},{"bytes":151,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},{"bytes":231,"matches":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},{"bytes":154,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},{"bytes":138,"matches":[{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},{"bytes":127,"matches":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},{"bytes":616,"matches":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},{"bytes":1190,"matches":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},{"bytes":223,"matches":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},{"bytes":803,"matches":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},{"bytes":131,"matches":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},{"bytes":301,"matches":[{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},{"bytes":89,"matches":[{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f"},{"bytes":8444,"matches":[{"line":4,"text":"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')"},{"line":64,"text":" return f\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\n\""},{"line":82,"text":" old=<REDACTED>"},{"line":97,"text":"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)"},{"line":99,"text":"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))"}],"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a"},{"bytes":28599,"matches":[{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE =<REDACTED>"},{"line":14,"text":"COOKIE =<REDACTED>"},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":20,"text":"    return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":73,"text":"    return f\"\"\"<!doctype html><html lang=<REDACTED>"},{"line":152,"text":"    return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":220,"text":"    page = f\"\"\"<!doctype html><html lang=\"ru\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><meta http-equiv=\"refresh\" content=\"30\"><title>Cluster Admin Incident Engine</title><style>body{{margin:0;background:#07101d;color:#e8eef7;font:14px Arial}}header{{padding:14px 22px;background:#101b2b;border-bottom:1px solid #2a3b54;display:flex;justify-content:space-between}}main{{max-width:1500px;margin:auto;padding:20px}}.cards{{display:grid;grid-template-columns:repeat(4,1fr);gap:14px}}.card,.panel{{background:#111d2e;border:1px solid #283b56;border-radius:14px;padding:16px}}.card b{{font-size:30px;display:block}}.card span,.muted{{color:#879ab3}}.grid{{display:grid;grid-template-columns:1fr 1fr;gap:14px;margin-top:14px}}.wide{{grid-column:1/-1}}table{{width:100%;border-collapse:collapse}}th,td{{padding:9px;border-bottom:1px solid #253750;text-align:left}}th{{color:#8fa3bc}}.state{{display:inline-flex;align-items:center;gap:7px;font-weight:700}}.state-dot{{width:10px;height:10px;border-radius:50%;display:inline-block;background:#8392a5}}.state-ok .state-dot{{background:#39d98a;box-shadow:0 0 12px rgba(57,217,138,.75)}}.state-bad .state-dot{{background:#ff5d73;box-shadow:0 0 12px rgba(255,93,115,.75)}}.state-warn .state-dot{{background:#f6c85f;box-shadow:0 0 12px rgba(246,200,95,.7)}}.state-unknown .state-dot{{background:#8392a5}}.row-bad{{background:rgba(255,93,115,.07)}}.row-warn{{background:rgba(246,200,95,.06)}}.card-health{{float:right}}.health-section{{padding:0;overflow:hidden}}.health-section summary{{list-style:none;cursor:pointer;display:flex;align-items:center;justify-content:space-between;gap:12px;padding:16px;user-select:none}}.health-section summary::-webkit-details-marker{{display:none}}.section-heading{{display:flex;align-items:center;gap:10px;font-size:20px;font-weight:700}}.section-heading .state{{font-size:0}}.section-title{{color:#e8eef7}}.section-count{{font-size:12px;color:#879ab3;border:1px solid #334a68;border-radius:999px;padding:3px 8px}}.section-chevron{{font-size:20px;color:#8fa3bc;transition:transform .18s ease}}.health-section[open] .section-chevron{{transform:rotate(180deg)}}.health-section[open] summary{{border-bottom:1px solid #253750;margin-bottom:8px}}.section-body{{padding:0 16px 16px}}button{{background:#18283d;color:white;border:1px solid #3b506d;border-radius:8px;padding:8px 12px}}@media(max-width:800px){{.cards,.grid{{grid-template-columns:1fr 1fr}}.wide{{grid-column:1/-1}}}}</style></head><body><header><div><b>Cluster Admin Incident Engine</b> {engine_badge}<div class=muted>observe/notify · auto-heal отключён</div></div><form method=\"post\" action=\"/logout\"><input type=\"hidden\" name=\"csrf\" value=\"{csrf}\"><button>Выйти</button></form></header><main><section class=cards>{cards}</section><section class=grid>{nodes_html}{incidents_html}{services_html}</section></main></body></html>\"\"\""},{"line":243,"text":"        \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":"        \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":"        \"cluster_admin_up 1\","},{"line":246,"text":"        \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":"        \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":"        \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":"        \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":"        \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":"        \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":"        \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":"        \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":"        \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":"        \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":"        \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":"        \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":"        \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":"        \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":"        \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":"        \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":"        \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":"        \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":267,"text":"        lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6"},{"bytes":922,"matches":[{"line":2,"text":"first = getpass.getpass(\"Новый пароль Cluster Admin: \")"},{"line":12,"text":"directory = \"/etc/cluster-admin-incident-engine\""},{"line":16,"text":"account = pwd.getpwnam(\"clusteradmin\")"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3"},{"bytes":10640,"matches":[{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":36,"text":"    data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":"    connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":41,"text":"        cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":50,"text":"            if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":57,"text":"        cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":108,"text":"    line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe"},{"bytes":1857,"matches":[{"line":3,"text":"  \"type\": \"cluster-admin-stage1-foundation-migration-plan\","},{"line":43,"text":"    \"function\": \"cluster_admin_enrich_collector_run\","}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670"},{"bytes":1342,"matches":[{"line":3,"text":"  \"collector_call\": \"SELECT cluster_admin_sync_foundation(%s)\","},{"line":15,"text":"  \"function\": \"cluster_admin_sync_foundation\","},{"line":16,"text":"  \"function_signature\": \"cluster_admin_sync_foundation(bigint)\","},{"line":26,"text":"  \"type\": \"cluster-admin-stage2-migration-plan\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c"},{"bytes":2355,"matches":[{"line":51,"text":"  \"type\": \"cluster-admin-stage2-dynamic-acceptance-contract\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588"},{"bytes":2832,"matches":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},{"bytes":3471,"matches":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},{"bytes":1433,"matches":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},{"bytes":848,"matches":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":10353,"matches":[{"line":7,"text":"<title>Homelab Cluster Admin</title>"},{"line":19,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":24,"text":"<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>"},{"line":28,"text":"<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"},{"line":29,"text":"<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>"},{"line":32,"text":"<div class=<REDACTED>"},{"line":33,"text":"<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>"},{"line":34,"text":"<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>"},{"line":35,"text":"<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":46,"text":"OK severity=<REDACTED>"},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a"},{"bytes":168,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify"}],"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729"},{"bytes":259,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741"},{"bytes":134,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517"},{"bytes":141,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6"},{"bytes":189,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941"},{"bytes":131,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0"}],"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":199,"matches":[{"line":1,"text":"STATUS=<REDACTED>"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680"},{"bytes":272,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec"},{"bytes":380,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"}],"gate":{"application_candidate_count":40,"ready_for_remote_transaction_design":true,"reference_file_count":1,"unit_count":16},"health_input_files":[{"bytes":380,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"},{"bytes":128,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac"},{"bytes":1535,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b"}],"indexes":[{"bytes":10353,"path":"/var/www/homelab-cluster-admin/index.html","root":"/var/www/homelab-cluster-admin","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a","title_lines":["<title>Homelab Cluster Admin</title>","<h1>Homelab Cluster Admin</h1>","<div class=\"card\"><h2>Self</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","<div class=\"card\"><h2>Observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md</pre></div>","<div class=\"card\"><h2>Full observer</h2><pre>STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md</pre></div>","<div class=\"card\"><h2>Mail Cloud backup</h2><pre>STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md</pre></div>","<div class=<REDACTED>","<div class=\"card\"><h2>Final readiness</h2><pre>STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md</pre></div>","<div class=\"card\"><h2>Strict seal</h2><pre>STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md</pre></div>","<div class=\"card\"><h2>Service map tail</h2><pre># Homelab cluster-admin full observer","pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"]}],"listeners":{"rc":0,"stderr":"","stdout":"State  Recv-Q Send-Q Local Address:Port Peer Address:PortProcess                                   \nLISTEN 0      5            0.0.0.0:8080      0.0.0.0:*    users:((\"python3\",pid=439,fd=3))         \nLISTEN 0      2048         0.0.0.0:8081      0.0.0.0:*    users:((\"python3\",pid=487,fd=6))         \nLISTEN 0      244        127.0.0.1:5432      0.0.0.0:*    users:((\"postgres\",pid=469,fd=6))        \nLISTEN 0      4096   127.0.0.53%lo:53        0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=18))\nLISTEN 0      4096         0.0.0.0:5355      0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=12))\nLISTEN 0      4096      127.0.0.54:53        0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=20))\nLISTEN 0      128          0.0.0.0:22        0.0.0.0:*    users:((\"sshd\",pid=457,fd=3))            \nLISTEN 0      244            [::1]:5432         [::]:*    users:((\"postgres\",pid=469,fd=5))        \nLISTEN 0      4096            [::]:5355         [::]:*    users:((\"systemd-resolve\",pid=380,fd=14))\nLISTEN 0      128             [::]:22           [::]:*    users:((\"sshd\",pid=457,fd=4))            "},"process_lines":["    439       1 root     root     python3         /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin","    487       1 cluster+ cluster+ python3         /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1"],"reference_files":[{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"<title>Homelab Cluster Admin</title>"},{"line":58,"text":"<h1>Homelab Cluster Admin</h1>"},{"line":74,"text":"<div class=\"card\"><h2>Service map tail</h2><pre>$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)</pre></div>"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"}],"units":[{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","Group":"clusteradmin","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","Group":"","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","Group":"root","LoadState":"loaded","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","User":"root","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}]},"v15_exact_result":{"changes_made":false,"control_plane_error":null,"declared_report_bytes":85056,"declared_report_sha256":"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf","identity_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":85940,"output_sha256":"c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8","output_source":"runner_json","rollback_restored":null,"rollback_started":false,"runner_document_rc":0,"runner_json_sha256":"6769d194cb102deb6fc37293d887af601bd012448bac4de49dd4bca3301f34a7","runner_rc":0,"runner_status":"OK","runner_text_sha256":"c68484da49e0164adaccfe9ecf92120c53d9324b2c3dcb96b4088376d3d177d2","runner_wrapper_matches_document":true,"sanitized_report_bytes":85006,"sanitized_report_sha256":"f05595551c37c00d473df29113cf9d597b2d3467ad5a2b50e8803e8579accc82","sanitizer_byte_delta":50,"sanitizer_changed_report":true,"state_document_sha256":"f438ecd342026e3db3f910ff87aba2e3c746147cf3fa64217fc4317774055874","state_path_sha256":"993095398c2c97878a69cd2e0aaa11e2ae1f587df8c8331835b5fd7ab7aa9de7","state_status":"OK"}}
+HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z","command_rc":0,"mutation_outcome":"NO_MUTATION","output_truncated_in_json":false,"rc":0,"rollback_restored":null,"rollback_started":false,"status":"OK","version":1}
+{"api":{"fetch_failure_count":0,"fetched_file_count":104,"fetched_total_bytes":297443,"route_coverage":{"blob":true,"branch":true,"repo":true,"tree":true},"text_file_count":104,"tree":{"blob_count":125,"body_emitted":false,"complete":true,"entry_count":161,"exact_commit":"fe1221b06643db3b00a621b0230f92a46a4edc2b","json_parse_ok":true,"status":200,"tree_count":36,"truncated":false}},"authorization_sent":true,"authorization_values_emitted":false,"branch":{"body_emitted":false,"branch":"main","commit_sha":"fe1221b06643db3b00a621b0230f92a46a4edc2b","json_parse_ok":true,"matches_expected_commit":true,"status":200},"changes_made":false,"command_id":"HOMELAB-CLUSTER-KB-V1-GITEA-DEFINED-CLUSTER-FACT-GAP-READ-ONLY-20260805T131000Z","command_rc":0,"credential_values_emitted":false,"endpoint_summary":{"address_literal_occurrence_count":81,"raw_addresses_emitted":false,"raw_url_occurrence_count":6,"raw_urls_emitted":false,"scope_counts":{"external_hostname":2,"known_external_gitea":1,"private":1},"tokens":[{"host_scope":"external_hostname","host_sha256":"8005522570737cc57f67406b7c6fb5d572797e4ba947dfb0bf3befee3639e667","path_depth":2,"port":null,"scheme":"https"},{"host_scope":"known_external_gitea","host_sha256":"92feea413ab6bd72fdd8b9102d872777d284d002a6be3e5323ca6e848105ee88","path_depth":2,"port":null,"scheme":"https"},{"host_scope":"external_hostname","host_sha256":"11d6a89894efa93b767088253526b02bac35874c1f63783507a3fb1d3b194e8a","path_depth":3,"port":null,"scheme":"https"},{"host_scope":"private","host_sha256":"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69","path_depth":1,"port":8081,"scheme":"http"}],"unique_endpoint_token_count":4},"error_fingerprint":"dadccd2e354a457c337b9089047ec0a0918a5d3cccd5a188b83f8dce425a59ab","error_registry":{"content_sha256":"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64","duplicate_fingerprint_count":0,"duplicate_id_count":0,"duplicate_ids":[],"fingerprint_value_count":0,"fingerprint_values_emitted":false,"migratable_record_count":0,"migration_coverage_ratio":0.0,"path":"errors/regression-cases.json","present":true,"prevention_evidence_count":0,"raw_records_emitted":false,"record_count":59,"status_counts":{},"unique_fingerprint_count":0,"unique_id_count":59},"fact_record_summary":{"domain_record_counts":{"access":10,"freshness":25,"identity":16},"emitted_record_count":25,"raw_record_values_emitted":false,"secret_hashes_emitted":false,"source_record_counts":{"inventory/access-paths.json":9,"inventory/cluster-nodes.json":3,"observed/cluster-guests.json":11,"observed/current-context.json":2},"top_fields":["status","id","node","type","name","vmid","address","service"],"unique_record_count":25},"fact_records":[],"fact_records_omitted_but_count_retained":true,"files_written":false,"http_methods":["GET"],"mutation_blueprint":{"atomicity_requirements":["read all exact source blobs again at the expected branch commit","build the full candidate tree in a temporary directory","validate JSON, JSONL, schemas, gate self-tests and repository tests","create byte-for-byte backup objects for every modified blob","create one commit against the expected parent commit","verify the new commit tree and CI-visible files","on any failure restore the exact previous branch state"],"code_root_choice":{"basis":"existing_top_level_code_or_directory","existing_python_file_count":2,"path":"tools/pre_command_gate.py","path_exists":false,"root":"tools"},"files_to_create_or_replace":["docs/CLUSTER-HANDBOOK.md","inventory/cluster-reference.json","docs/ERROR-SYSTEM-V2.md","schemas/error-record.schema.json","errors/error-registry.jsonl","tools/pre_command_gate.py","tests/test_error_system_v2.py"],"files_to_update_conditionally":[".gitea/workflows/ci.yml","docs/WORKFLOW.md"],"migration_requirements":{"append_runtime_failures_from_known_error_gate":true,"derive_fingerprint_only_from_sanitized_stable_fields":true,"mark_parallel_draft_superseded_only_after_validation":true,"preserve_all_existing_records":true,"require_prevention_test_for_each_active_record":true},"migration_source":"errors/regression-cases.json","ready":true,"transaction_preconditions":{"branch":"main","exact_source_sha256":{".gitea/workflows/ci.yml":"b078ac1b57e0846a90fd6acb1f3cb0866dbed0fb3bbde0d7070d29a344907681","docs/WORKFLOW.md":"87a6f9a0960ee7240261138ecea122fd8fbd727d87bb0b27b02c103c3bc2f122","errors/regression-cases.json":"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64","schemas/change.schema.json":"57fbce7c6ca1f53c5ec2f8cd604d787f01019a2184b611e3b4406b467331d0af","schemas/context.schema.json":"84c3c29984402b25c102575b54bdb82544844e455d3d2e115fefc5f42021e043"},"expected_branch_commit":"fe1221b06643db3b00a621b0230f92a46a4edc2b","proposed_paths":[{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"docs/CLUSTER-HANDBOOK.md"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"inventory/cluster-reference.json"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"docs/ERROR-SYSTEM-V2.md"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"schemas/error-record.schema.json"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"errors/error-registry.jsonl"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"tests/test_error_system_v2.py"},{"creation_precondition":"must_remain_absent","exists_at_exact_commit":false,"path":"tools/pre_command_gate.py"}],"repository":"homelab-admin/homelab-ops"}},"mutation_outcome":"NO_MUTATION","network_requests_made":true,"next_action":"RUN_ONE_IDEMPOTENT_GITEA_TRANSACTION_CREATE_CLUSTER_HANDBOOK_AND_ERROR_SYSTEM_V2","output_truncated_in_json":false,"pre_command_known_error_gate":{"checked":true,"known_error_count":28,"known_error_ids":["candidate-file-limit-600-20260805","runner-wrapper-mismatch-after-success-20260805","canonical-cluster-handbook-ambiguous-20260805","pvesh-adapter-context-unreadable-20260805","homelab-ops-bare-repository-not-found-local-20260805","gitea-network-fallback-stage-not-emitted-20260805","gitea-owner-hardcoded-homelab-admin-20260805","gitea-git-auth-required-or-credential-unavailable-20260805","gitea-process-self-match-payload-name-20260805","sanitize-bool-attributeerror-20260805","reader-v3-path-misclassified-as-route-20260805","reader-v3-loopback-assumption-20260805","reader-v3-capability-not-treated-as-secret-20260805","reader-v3-systemd-unit-fed-to-python-ast-20260805","reader-v3-helper-control-flow-not-traced-20260805","reader-v3-post-capability-operation-not-derived-20260805","reader-v3-detected-operations-not-tested-20260805","reader-v3-manifest-global-not-bound-in-sandbox-20260805","reader-v3-result-file-second-source-not-read-20260805","reader-v3-negative-control-mixed-with-production-20260805","reader-v3-manifest-schema-guessed-20260805","reader-result-output-bound-after-compaction-20260805","reader-v3-generation-record-shape-guessed-20260805","reader-v3-result-file-assumed-standalone-json-20260805","reader-v3-gitea-detail-marker-false-inventory-20260805","reverse-proxy-runtime-not-visible-on-host-20260805","gitea-public-owner-not-anonymously-discoverable-20260805","exact-output-long-line-not-projectable-by-portal-view-20260805"],"previous_exact_identity_verified":true},"previous_exact_verification":{"bundle_bytes_ok":true,"bundle_sha256_ok":true,"capsule_member_match_count":0,"capsule_sha256_basis":"verified exact lane before command publication; optional bundle member count reported separately","checks":{"changes_made_false":true,"command_id":true,"command_rc_zero":true,"contract_ok":true,"mutation_outcome_no_mutation":true,"projection_ok":true,"rollback_started_false":true,"runner_rc_zero":true,"runner_status_ok":true},"embedded_output_match_count":1,"exists":true,"identity_fully_verified":true,"member_count":6,"path_from_exact_result":true,"runner_json_match_count":1,"runner_text_match_count":1,"unsafe_member_rejected":false},"private_addresses_emitted":false,"raw_repository_contents_emitted":false,"rc":0,"read_only":true,"ready_for_idempotent_mutation":true,"regression_tests":{"endpoint_token_hides_host":true,"git_blob_oid_semantics":true,"known_error_gate_complete":true,"passed":true,"private_address_not_emitted":true,"safe_record_fingerprint_deterministic":true,"sanitize_bool_type_safe":true,"secret_value_redacted_before_record_hash":true,"strict_handbook_excludes_error_registry":true},"repository":{"archived":false,"default_branch":"main","empty":false,"full_name":"homelab-admin/homelab-ops","identity_safe":true,"mirror":false,"name":"homelab-ops","owner":"homelab-admin","private":true,"size":3837},"result_projection":{"budget_bytes":24976,"bytes":24326,"steps":["source_summaries_24","fact_records_32","drop_json_shapes_and_limit_headings","handbook_candidates_4","omit_fact_records"]},"rollback_restored":null,"rollback_started":false,"root_cause_classification":"GITEA_CLUSTER_FACTS_AND_DOCUMENT_GAPS_EXACTLY_ANALYZED_MUTATION_READY","schema":1,"secret_hashes_emitted":false,"source_summaries":[{"blob_sha":"7a8cb9e2dd5bc1235937aa211293b47ed4e26263","bytes":29485,"content_sha256":"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64","domains":{"access":9,"backup":22,"error_system":38,"freshness":5,"internal_external":1,"repair":38,"restore":9,"services":8},"headings":[],"json_parse_ok":true,"path":"errors/regression-cases.json","raw_content_emitted":false,"secret_key_name_occurrence_count":60},{"blob_sha":"bad7037197f12ce9ce0b8be3b5723345699680b5","bytes":2289,"content_sha256":"e07ccfac39f862c2bce2da0d31112e8e055ca9d3f23a01a8f33302863239203a","domains":{"access":1,"backup":0,"error_system":1,"freshness":1,"internal_external":2,"repair":0,"restore":0,"services":0},"headings":[],"json_parse_ok":true,"path":"changes/CR-2026-0018/regression-cases-draft.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"d072c291ee6b99466be0f4897e9d1308ee02e2b2","bytes":1134,"content_sha256":"df400edc01d8201665c92e7434f8b0663eac0d34aab35e918abfb9bb873877e8","domains":{"access":0,"backup":0,"error_system":0,"freshness":2,"internal_external":0,"repair":1,"restore":0,"services":4},"headings":["Caller inventory requirements","Search surfaces","Required record","Gate"],"json_parse_ok":false,"path":"changes/CR-2026-0018/caller-inventory-requirements.md","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"823d591344d8b8f67e213e2b890acdbd7bfc9864","bytes":211,"content_sha256":"d3347712264b62c8768c952575574140c388df6d4d045c824ed9a8722bad1454","domains":{"access":0,"backup":0,"error_system":0,"freshness":0,"internal_external":0,"repair":1,"restore":0,"services":0},"headings":["Changes"],"json_parse_ok":false,"path":"changes/README.md","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"c42de139cdf529611d50ad86291fb40df5bc0d09","bytes":1652,"content_sha256":"ac5dc881dcd9fcb72644bc893e866f463c9c2b9b4544e32434e21b5d34168fb5","domains":{"access":9,"backup":0,"error_system":0,"freshness":0,"internal_external":0,"repair":0,"restore":0,"services":0},"headings":[],"json_parse_ok":true,"path":"inventory/access-paths.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"5a1fb00b587a813cee2a7bd98be451bd22173f5e","bytes":4952,"content_sha256":"6b732f5ce3af0e8066d020bf10ee81b764b6d93ee5f08a9c8f17d0075eeced32","domains":{"access":0,"backup":0,"error_system":0,"freshness":0,"internal_external":0,"repair":0,"restore":1,"services":22},"headings":[],"json_parse_ok":true,"path":"observed/cluster-guests.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"3327f8b644b3f3e83774d5e7748a8dfa6b4f1ebf","bytes":958,"content_sha256":"5e7236f3b8072ff6115ab5bc50ad448425882e86463e90d85df1a1c8eb502d65","domains":{"access":0,"backup":1,"error_system":0,"freshness":0,"internal_external":0,"repair":1,"restore":0,"services":2},"headings":["Архитектура управления","Разделение ответственности"],"json_parse_ok":false,"path":"docs/ARCHITECTURE.md","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"3b034041ab628d15e74a7167b5545b366aeb4ad1","bytes":526,"content_sha256":"87a6f9a0960ee7240261138ecea122fd8fbd727d87bb0b27b02c103c3bc2f122","domains":{"access":0,"backup":0,"error_system":0,"freshness":1,"internal_external":0,"repair":1,"restore":0,"services":0},"headings":["Change workflow"],"json_parse_ok":false,"path":"docs/WORKFLOW.md","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"f03826dcf6f8c6af4a8a78206c1b841ffaffd2ca","bytes":3020,"content_sha256":"91c66d8ee6622d925174055599ad5ece6270c35f8d7d7967a26609f7e290c7e6","domains":{"access":0,"backup":1,"error_system":3,"freshness":4,"internal_external":0,"repair":1,"restore":1,"services":0},"headings":["Одна постоянная команда","Что проверяется автоматически","Правила безопасности"],"json_parse_ok":false,"path":"docs/skladchik-cookie-refresh.md","raw_content_emitted":false,"secret_key_name_occurrence_count":17},{"blob_sha":"b4979003f9bb899f4a5de41ede47eacaeac48dee","bytes":2264,"content_sha256":"e74454f528cdb5522e2c43d959472a04b7cc36e33da7aa5a5bd664fc79015344","domains":{"access":2,"backup":0,"error_system":0,"freshness":2,"internal_external":0,"repair":0,"restore":0,"services":1},"headings":["Skladchik Moderator Assistant","Назначение","Что планируется подготовить позднее","Что сейчас не делать"],"json_parse_ok":false,"path":"docs/planned/skladchik-moderator-assistant.md","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"9586f62339cc657d18d033c104229458ebcadb58","bytes":1290,"content_sha256":"cfb21fbf6ead4b3537b5f6f43e19602365b7b16adba5f317e48f8f19f5d40475","domains":{"access":0,"backup":0,"error_system":2,"freshness":0,"internal_external":1,"repair":1,"restore":0,"services":0},"headings":[],"json_parse_ok":true,"path":"policies/operating-policy.json","raw_content_emitted":false,"secret_key_name_occurrence_count":1},{"blob_sha":"2032d3ebab4794625854b8c80d23ab72aac9998b","bytes":1436,"content_sha256":"e06b7e6285eda2885ef913aa954a70abc7c1d892199a8b7cea2ada98b8edba34","domains":{"access":0,"backup":2,"error_system":3,"freshness":0,"internal_external":0,"repair":3,"restore":2,"services":0},"headings":[],"json_parse_ok":true,"path":"policies/cluster-cloud-quorum.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"8352f9a2024afea21049cd7700e460ee096c6c53","bytes":969,"content_sha256":"57fbce7c6ca1f53c5ec2f8cd604d787f01019a2184b611e3b4406b467331d0af","domains":{"access":0,"backup":2,"error_system":0,"freshness":0,"internal_external":0,"repair":2,"restore":0,"services":0},"headings":[],"json_parse_ok":true,"path":"schemas/change.schema.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"7dc9476a8ca0227b4d5bf0cbd2c706c60af21119","bytes":539,"content_sha256":"84c3c29984402b25c102575b54bdb82544844e455d3d2e115fefc5f42021e043","domains":{"access":0,"backup":0,"error_system":0,"freshness":0,"internal_external":0,"repair":0,"restore":0,"services":0},"headings":[],"json_parse_ok":true,"path":"schemas/context.schema.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"3d6801bd6e7faefd1fc4579c41dc387f3dd1a9f0","bytes":318,"content_sha256":"b078ac1b57e0846a90fd6acb1f3cb0866dbed0fb3bbde0d7070d29a344907681","domains":{"access":0,"backup":0,"error_system":1,"freshness":0,"internal_external":0,"repair":0,"restore":0,"services":1},"headings":[],"json_parse_ok":false,"path":".gitea/workflows/ci.yml","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"68ae87029a1660255df02ff192e5b9206e9a6c05","bytes":13579,"content_sha256":"d3108777e5094468aa3e633740ce8a56c90ca957a9a62932be896bbe784680b0","domains":{"access":22,"backup":23,"error_system":16,"freshness":3,"internal_external":1,"repair":22,"restore":10,"services":13},"headings":[],"json_parse_ok":true,"path":"tasks/skladchik-reports-monitor-controlled-reauth-v1/task.json","raw_content_emitted":false,"secret_key_name_occurrence_count":43},{"blob_sha":"6e22f5ca60d95965765c03ac1d012631aaae68b0","bytes":6750,"content_sha256":"3a8fdfb6b4b64880012fcc3bc6fee4be6f101377310325541f9776de6d1623f7","domains":{"access":0,"backup":4,"error_system":16,"freshness":4,"internal_external":2,"repair":18,"restore":1,"services":28},"headings":[],"json_parse_ok":true,"path":"tasks/homelab-logrotate-namespace-latch-clearance-v1/task.json","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"a1528e7cbc2b1027bfd7faa220bf1f096cdde1f5","bytes":3665,"content_sha256":"666072447ba11d346772f62f274dadc974c4463e20f67e67f82f228f1fcf2ce8","domains":{"access":20,"backup":17,"error_system":1,"freshness":1,"internal_external":0,"repair":4,"restore":10,"services":7},"headings":[],"json_parse_ok":false,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/rollback.sh","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"d2adf1bf345063da2b02040697f4358ba4ca4ae1","bytes":4667,"content_sha256":"a49bcb32ac7212d7fb9768db8a277cb5525c873c84dd6fb633edae35dc6de532","domains":{"access":8,"backup":14,"error_system":9,"freshness":1,"internal_external":0,"repair":1,"restore":6,"services":10},"headings":[],"json_parse_ok":false,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"c6f94aa845bc4cd8fc061e5e03e0fd7771a01bc6","bytes":7440,"content_sha256":"636254d2526f0f62184d47cdb4c30c2831b11869c2000c557666f60b55d7691b","domains":{"access":13,"backup":24,"error_system":13,"freshness":0,"internal_external":0,"repair":12,"restore":4,"services":8},"headings":[],"json_parse_ok":true,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/task.json","raw_content_emitted":false,"secret_key_name_occurrence_count":2},{"blob_sha":"9cc7d2fa3bc044d311500a5cf798273c79e39c35","bytes":4233,"content_sha256":"ac0885c9762b99f360256ad3b3894f9cbcc4a8deab529a19ef264fe446bff029","domains":{"access":8,"backup":16,"error_system":8,"freshness":1,"internal_external":0,"repair":7,"restore":0,"services":11},"headings":[],"json_parse_ok":false,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/verify.sh","raw_content_emitted":false,"secret_key_name_occurrence_count":0},{"blob_sha":"774e2b0e591c9ada9aac9700a1b8427645284f1e","bytes":4665,"content_sha256":"34de7dd0cc60c90c1b74e7a91334cb737d33c761882d1483d5b39f15b439c6f1","domains":{"access":13,"backup":19,"error_system":1,"freshness":1,"internal_external":0,"repair":1,"restore":0,"services":7},"headings":[],"json_parse_ok":false,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh","raw_content_emitted":false,"secret_key_name_occurrence_count":3},{"blob_sha":"938db71d80350ac328ec90fe7bb5176a4d6d50ac","bytes":2867,"content_sha256":"3b8687392163cf5b6f8b0874087320fe74ceaa5fee897dd0e7762d8c71476a1c","domains":{"access":4,"backup":4,"error_system":1,"freshness":0,"internal_external":0,"repair":8,"restore":11,"services":3},"headings":[],"json_parse_ok":false,"path":"tasks/skladchik-reports-monitor-controlled-reauth-v1/phases/rollback.sh","raw_content_emitted":false,"secret_key_name_occurrence_count":2},{"blob_sha":"aefe7ac6860ec2107c281485b35c8cba1f1196ef","bytes":4846,"content_sha256":"cfde805604d58d87d975584431aeb024bef21e863c4293d4726e5504415fe9bc","domains":{"access":0,"backup":6,"error_system":5,"freshness":2,"internal_external":1,"repair":3,"restore":9,"services":0},"headings":[],"json_parse_ok":false,"path":"tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py","raw_content_emitted":false,"secret_key_name_occurrence_count":21}],"status":"OK","strict_handbook":{"candidates":[{"canonical_phrase_hits":0,"content_sha256":"91c66d8ee6622d925174055599ad5ece6270c35f8d7d7967a26609f7e290c7e6","covered_domains":["backup","error_system","repair","restore"],"headings":["Одна постоянная команда","Что проверяется автоматически","Правила безопасности"],"path":"docs/skladchik-cookie-refresh.md","score":80,"title_signal":false},{"canonical_phrase_hits":1,"content_sha256":"5e7236f3b8072ff6115ab5bc50ad448425882e86463e90d85df1a1c8eb502d65","covered_domains":["backup","repair","services"],"headings":["Архитектура управления","Разделение ответственности"],"path":"docs/ARCHITECTURE.md","score":75,"title_signal":false},{"canonical_phrase_hits":1,"content_sha256":"d9cb129e009176a7dde2a07f59760acddd9e17237eef563e97a55b3e62838bd1","covered_domains":["repair"],"headings":["homelab-ops","Порядок работы","Локальная проверка"],"path":"README.md","score":55,"title_signal":true},{"canonical_phrase_hits":0,"content_sha256":"e74454f528cdb5522e2c43d959472a04b7cc36e33da7aa5a5bd664fc79015344","covered_domains":["access","services"],"headings":["Skladchik Moderator Assistant","Назначение","Что планируется подготовить позднее","Что сейчас не делать","Условие возобновления"],"path":"docs/planned/skladchik-moderator-assistant.md","score":40,"title_signal":false}],"path":null,"previous_loose_candidate_rejected":"errors/regression-cases.json","qualified_count":0,"rejection_reason":"error registry is not a cluster handbook under the strict source-class definition","status":"NOT_FOUND","strict_definition":{"allowed_source_classes":["docs","inventory","root index"],"excluded_source_classes":["errors","changes","tasks","tests","workflows","schemas"],"minimum_domains":6,"requires_canonical_phrase":true,"requires_cluster_title":true}},"verified_gaps":[{"evidence":{"qualified_count":0,"strict_status":"NOT_FOUND"},"gap_id":"canonical_cluster_handbook_absent","remediation":"create one canonical docs/CLUSTER-HANDBOOK.md and link every fact to an exact machine-readable source","severity":"BLOCKING"},{"evidence":{"path_verified_absent":true},"gap_id":"machine_readable_cluster_reference_absent","remediation":"create inventory/cluster-reference.json with source_sha256, verified_at and stale/unknown states","severity":"BLOCKING"},{"evidence":{"record_count":0},"gap_id":"structured_backup_restore_facts_absent","remediation":"materialize backup_restore facts into the machine-readable cluster reference","severity":"HIGH"},{"evidence":{"record_count":0},"gap_id":"structured_repair_facts_absent","remediation":"materialize repair facts into the machine-readable cluster reference","severity":"HIGH"},{"evidence":{"record_count":0},"gap_id":"structured_scope_facts_absent","remediation":"materialize scope facts into the machine-readable cluster reference","severity":"HIGH"},{"evidence":{"error_record_count":59,"fingerprint_count":0},"gap_id":"error_registry_has_no_fingerprints","remediation":"seed stable normalized fingerprints and block duplicate failed fingerprints before publication","severity":"BLOCKING"},{"evidence":{"error_record_count":59,"prevention_evidence_count":0},"gap_id":"error_registry_has_no_prevention_evidence","remediation":"require one positive and one negative prevention test for every active error record","severity":"BLOCKING"},{"evidence":{"ci_mentions_fingerprint":false,"ci_mentions_pre_command_gate":false},"gap_id":"ci_does_not_enforce_error_fingerprint_gate","remediation":"wire the pre-command gate and registry/schema tests into .gitea/workflows/ci.yml","severity":"BLOCKING"},{"evidence":{"both_present":true,"content_sha256_equal":false},"gap_id":"parallel_draft_and_canonical_error_registries_diverge","remediation":"declare canonical ownership and mark the draft superseded after migration","severity":"MEDIUM"},{"evidence":{"matching_path_count":0},"gap_id":"superseded_command_registry_absent","remediation":"record superseded command_id/capsule/result relationships during post-result ingestion","severity":"HIGH"}],"workflow_gate_signals":{"ci_mentions_error_registry":false,"ci_mentions_fingerprint":false,"ci_mentions_pre_command_gate":false,"ci_present":true,"repository_mentions_duplicate_failed_command_blocked":false,"repository_mentions_error_register_checked":false,"repository_mentions_reference_register_checked":false}}
 
 OUTPUT_END
 CHAT_OUTPUT_END