From a1fb7372738025faedd677ee7da612c996b313e5 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Wed, 5 Aug 2026 13:08:02 +0000 Subject: [PATCH] runtime: publish HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z --- ...PROJECTION-READ-ONLY-20260805T130500Z.json | 38 +++++++++++++++++++ ...-PROJECTION-READ-ONLY-20260805T130500Z.txt | 34 +++++++++++++++++ runtime/latest.json | 16 ++++---- runtime/latest.txt | 14 +++---- 4 files changed, 87 insertions(+), 15 deletions(-) create mode 100644 runtime/history/HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z.json create mode 100644 runtime/history/HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z.txt diff --git a/runtime/history/HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z.json b/runtime/history/HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z.json new file mode 100644 index 0000000..e399a6c --- /dev/null +++ b/runtime/history/HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "verify", + "component": "cluster-knowledge-base-error-system", + "started_at_utc": "2026-08-05T13:08:01Z", + "finished_at_utc": "2026-08-05T13:08:01Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83", + "command_sha256": "8d626b3d942e0c8ad8135e1fbf4148a82c5df736a52cc969eb0f6ae022f1ee59", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "change_declared": false, + "result_contract_valid": true, + "result_contract_status": null, + "result_contract_error": null, + "command_rc": 0, + "changes_made": false, + "rollback_started": false, + "rollback_restored": null, + "mutation_outcome": "NO_MUTATION", + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "2a7ec04b2e7036e6cc5751216e516b7d78ef56dc135326379580ef58dbc66a29", + "sanitized_output_sha256": "2a7ec04b2e7036e6cc5751216e516b7d78ef56dc135326379580ef58dbc66a29", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z\",\"command_rc\":0,\"mutation_outcome\":\"NO_MUTATION\",\"output_truncated_in_json\":false,\"rc\":0,\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\n{\"bounded_adapter_scan\":null,\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z\",\"command_rc\":0,\"evidence_bundle\":{\"bundle_bytes_ok\":true,\"bundle_sha256_ok\":true,\"exists\":true,\"member_count\":6,\"output_member_matches\":0,\"path_is_exact_from_previous_result\":true,\"runner_json_member_matches\":1,\"runner_text_member_matches\":1,\"unsafe_member_rejected\":false},\"files_written\":false,\"mutation_outcome\":\"NO_MUTATION\",\"network_requests_made\":false,\"next_action\":\"RUN_ONE_READ_ONLY_GITEA_DEFINED_CLUSTER_FACT_AND_DOCUMENT_GAP_ANALYSIS\",\"output_truncated_in_json\":false,\"pre_command_known_error_gate\":{\"checked\":true,\"count\":28,\"ids\":[\"candidate-file-limit-600-20260805\",\"runner-wrapper-mismatch-after-success-20260805\",\"canonical-cluster-handbook-ambiguous-20260805\",\"pvesh-adapter-context-unreadable-20260805\",\"homelab-ops-bare-repository-not-found-local-20260805\",\"gitea-network-fallback-stage-not-emitted-20260805\",\"gitea-owner-hardcoded-homelab-admin-20260805\",\"gitea-git-auth-required-or-credential-unavailable-20260805\",\"gitea-process-self-match-payload-name-20260805\",\"sanitize-bool-attributeerror-20260805\",\"reader-v3-path-misclassified-as-route-20260805\",\"reader-v3-loopback-assumption-20260805\",\"reader-v3-capability-not-treated-as-secret-20260805\",\"reader-v3-systemd-unit-fed-to-python-ast-20260805\",\"reader-v3-helper-control-flow-not-traced-20260805\",\"reader-v3-post-capability-operation-not-derived-20260805\",\"reader-v3-detected-operations-not-tested-20260805\",\"reader-v3-manifest-global-not-bound-in-sandbox-20260805\",\"reader-v3-result-file-second-source-not-read-20260805\",\"reader-v3-negative-control-mixed-with-production-20260805\",\"reader-v3-manifest-schema-guessed-20260805\",\"reader-result-output-bound-after-compaction-20260805\",\"reader-v3-generation-record-shape-guessed-20260805\",\"reader-v3-result-file-assumed-standalone-json-20260805\",\"reader-v3-gitea-detail-marker-false-inventory-20260805\",\"reverse-proxy-runtime-not-visible-on-host-20260805\",\"gitea-public-owner-not-anonymously-discoverable-20260805\",\"exact-output-long-line-not-projectable-by-portal-view-20260805\"]},\"projection\":{\"analysis_outcome\":{\"error_fingerprint\":\"26d65952ac8f64197a6fd76ec56e3b5ffda8c4838efe2af2947a603d01270b92\",\"next_action\":\"RUN_ONE_READ_ONLY_GITEA_DEFINED_CLUSTER_FACT_AND_DOCUMENT_GAP_ANALYSIS\",\"ready_for_gitea_defined_cluster_fact_analysis\":true,\"root_cause_classification\":\"GITEA_CLUSTER_HANDBOOK_AND_ERROR_SYSTEM_DEEPLY_ANALYZED\",\"status\":\"OK\"},\"canonical_handbook\":{\"candidates\":[{\"canonical_phrase_hits\":3,\"canonical_score\":227,\"content_sha256\":\"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64\",\"headings\":[],\"inbound_reference_count\":8,\"path\":\"errors/regression-cases.json\"},{\"canonical_phrase_hits\":0,\"canonical_score\":74,\"content_sha256\":\"a49bcb32ac7212d7fb9768db8a277cb5525c873c84dd6fb633edae35dc6de532\",\"headings\":[],\"inbound_reference_count\":0,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh\"},{\"canonical_phrase_hits\":1,\"canonical_score\":56,\"content_sha256\":\"ac5dc881dcd9fcb72644bc893e866f463c9c2b9b4544e32434e21b5d34168fb5\",\"headings\":[],\"inbound_reference_count\":1,\"path\":\"inventory/access-paths.json\"},{\"canonical_phrase_hits\":0,\"canonical_score\":55,\"content_sha256\":\"666072447ba11d346772f62f274dadc974c4463e20f67e67f82f228f1fcf2ce8\",\"headings\":[],\"inbound_reference_count\":0,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/rollback.sh\"},{\"canonical_phrase_hits\":0,\"canonical_score\":54,\"content_sha256\":\"636254d2526f0f62184d47cdb4c30c2831b11869c2000c557666f60b55d7691b\",\"headings\":[],\"inbound_reference_count\":0,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/task.json\"}],\"path\":\"errors/regression-cases.json\",\"status\":\"UNIQUE_CONFIRMED\"},\"documentation_coverage\":{\"access\":{\"file_count\":22,\"paths\":[\"changes/CR-2026-0018/regression-cases-draft.json\",\"docs/planned/skladchik-moderator-assistant.md\",\"errors/regression-cases.json\",\"inventory/access-paths.json\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/dry-run.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/prepare.sh\"]},\"backup\":{\"file_count\":29,\"paths\":[\"docs/ARCHITECTURE.md\",\"docs/skladchik-cookie-refresh.md\",\"errors/regression-cases.json\",\"observed/cluster-guests.json\",\"policies/cluster-cloud-quorum.json\",\"schemas/change.schema.json\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/assets/pve02-vm160-preflight-probe.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/assets/vm160-cloud-quorum-worker.sh\"]},\"error_system\":{\"file_count\":44,\"paths\":[\".gitea/workflows/ci.yml\",\"changes/CR-2026-0018/caller-inventory-requirements.md\",\"changes/CR-2026-0018/regression-cases-draft.json\",\"changes/README.md\",\"docs/ARCHITECTURE.md\",\"docs/skladchik-cookie-refresh.md\",\"docs/WORKFLOW.md\",\"errors/regression-cases.json\"]},\"internal_external\":{\"file_count\":7,\"paths\":[\"changes/CR-2026-0018/regression-cases-draft.json\",\"errors/regression-cases.json\",\"policies/operating-policy.json\",\"tasks/homelab-logrotate-namespace-latch-clearance-v1/README.txt\",\"tests/test_cr_2026_0005.py\",\"tests/test_cr_2026_0011_skladchik_cookie_jar_reset.py\",\"tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py\"]},\"repair\":{\"file_count\":31,\"paths\":[\"changes/CR-2026-0018/caller-inventory-requirements.md\",\"changes/README.md\",\"docs/ARCHITECTURE.md\",\"docs/skladchik-cookie-refresh.md\",\"docs/WORKFLOW.md\",\"errors/regression-cases.json\",\"policies/cluster-cloud-quorum.json\",\"policies/operating-policy.json\"]},\"services\":{\"file_count\":21,\"paths\":[\"changes/CR-2026-0018/caller-inventory-requirements.md\",\"docs/ARCHITECTURE.md\",\"docs/planned/skladchik-moderator-assistant.md\",\"errors/regression-cases.json\",\"observed/cluster-guests.json\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/assets/vm160-cloud-quorum-worker.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/dry-run.sh\"]}},\"enumeration\":{\"candidate_file_count\":50,\"fetch_failure_count\":0,\"fetched_file_count\":50,\"fetched_total_bytes\":173536,\"tree_blob_count\":125,\"tree_complete\":true,\"tree_directory_count\":36,\"tree_entry_count\":161},\"error_system_current\":{\"duplicate_error_ids\":[],\"duplicate_fingerprints\":[],\"enforcement_path_count\":7,\"enforcement_paths\":[\"schemas/change.schema.json\",\"schemas/context.schema.json\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/assets/pve02-vm160-preflight-probe.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh\",\"tasks/homelab-logrotate-namespace-latch-clearance-v1/phases/preflight.sh\",\"tasks/skladchik-reports-monitor-controlled-reauth-v1/assets/edge-preflight.sh\",\"tasks/skladchik-reports-monitor-controlled-reauth-v1/phases/preflight.sh\"],\"gate_signals\":{\"ci_workflow_present\":true,\"duplicate_failed_command_block_signal\":false,\"error_register_checked_signal\":false,\"fingerprint_signal\":false,\"preflight_references_error_register\":true,\"reference_register_checked_signal\":false,\"schema_validation_signal\":true,\"tests_present\":true},\"machine_parseable_registry_file_count\":2,\"prevention_evidence_count\":0,\"registry_path_count\":2,\"registry_paths\":[\"changes/CR-2026-0018/regression-cases-draft.json\",\"errors/regression-cases.json\"],\"regression_path_count\":2,\"regression_paths\":[\"changes/CR-2026-0018/regression-cases-draft.json\",\"errors/regression-cases.json\"],\"superseded_path_count\":0,\"superseded_paths\":[],\"test_path_count\":14,\"test_paths\":[\"tests/fixtures/gitea-legacy-auth-permission.json\",\"tests/test_cr_2026_0005.py\",\"tests/test_cr_2026_0006.py\",\"tests/test_cr_2026_0006_vm160_task.py\",\"tests/test_cr_2026_0007_skladchik_reauth.py\",\"tests/test_cr_2026_0007_skladchik_semantic_repair.py\",\"tests/test_cr_2026_0008_skladchik_snapshot_integrity.py\",\"tests/test_cr_2026_0009_skladchik_concurrency_guard.py\",\"tests/test_cr_2026_0011_skladchik_cookie_jar_reset.py\",\"tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py\"],\"unique_error_id_count\":61,\"unique_fingerprint_count\":0,\"workflow_path_count\":1,\"workflow_paths\":[\".gitea/workflows/ci.yml\"]},\"gaps\":[\"no_static_evidence_error_fingerprint_gate\"],\"pre_command_gate\":{\"checked\":true,\"projection_command_known_error_count\":28,\"regression_tests_passed\":true,\"reported_error_count\":27},\"previous_exact_result\":{\"capsule_sha256\":\"031aa6c1029c24d8b7dac8192707376fa67ee93cdcfceedcfb768a0b7d4ceb8e\",\"command_id\":\"HOMELAB-CLUSTER-KB-V1-GITEA-REPOSITORY-CONTENT-DEEP-ANALYSIS-READ-ONLY-20260805T124500Z\",\"identity_fully_verified\":true,\"output_bytes\":27219,\"output_sha256\":\"ead28fd156e646c27f6e988c12ea1e89a498882f07071faacba670e87c8a7983\",\"runner_json_sha256\":\"e3211267d27f7497339cd34ecdbe0cce21d039965625ffd5f3236dc20513b082\",\"runner_text_sha256\":\"083286377044254ff016f0f9024655e228a67ae2737d7e1d9641996d5af8e09a\"},\"previous_result_contract\":{\"changes_made\":false,\"command_rc\":0,\"mutation_outcome\":\"NO_MUTATION\",\"output_truncated_in_json\":false,\"rc\":0,\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\"},\"recommended_error_system_v2\":{\"acceptance_criteria\":[\"zero hard-coded owner/path/host/version unless proven by exact prior evidence or current Gitea content\",\"zero mutation without byte-for-byte backup and tested rollback\",\"zero repeated known error fingerprint\",\"all command outputs bounded and non-truncated\",\"all new errors recorded before any subsequent command\"],\"components\":[{\"component\":\"machine_error_registry\",\"requirement\":\"one append-only machine-readable record per proven failure with stable id, normalized fingerprint, exact command/capsule/result SHA, root cause, correction and prevention test\"},{\"component\":\"pre_command_gate\",\"requirement\":\"before every .hcmd, read canonical cluster reference and entire active error registry; block duplicate failed fingerprints and commands that violate a prevention test\"},{\"component\":\"transaction_manifest\",\"requirement\":\"every mutation declares exact source SHA, candidate SHA, byte backup SHA, preconditions, atomic apply, verification and deterministic rollback\"},{\"component\":\"post_result_ingestion\",\"requirement\":\"after exact-result verification, automatically classify the outcome, append new proven failures, close corrected cases and link superseded commands\"},{\"component\":\"regression_suite\",\"requirement\":\"positive and negative fixtures for every recorded error; payload self-test and repository CI must pass before command publication\"},{\"component\":\"knowledge_freshness\",\"requirement\":\"canonical handbook stores last verified fact source, verification time, source SHA and stale/unknown markers; commands may not rely on stale or unknown values\"}],\"grounded_in_detected_gaps\":[\"no_static_evidence_error_fingerprint_gate\"],\"minimum_gate_order\":[\"verify previous exact result identity\",\"read canonical handbook at exact branch commit\",\"read active errors and prevention tests\",\"validate target facts from authoritative source\",\"run payload regression tests\",\"only then publish or execute the next command\"],\"name\":\"Error System V2: evidence-first command gate\",\"reuse_existing_error_root\":\"changes/CR-2026-0018\"},\"repository\":{\"branch_commit_sha\":\"fe1221b06643db3b00a621b0230f92a46a4edc2b\",\"default_branch\":\"main\",\"full_name\":\"homelab-admin/homelab-ops\",\"private\":true},\"source_result_projection\":{\"budget_bytes\":29232,\"bytes\":26771,\"max_bytes\":30000,\"steps\":[\"top_analyzed_files_12\",\"handbook_candidates_6\"]}},\"projection_bytes_before_final_wrapper\":11730,\"rc\":0,\"read_only\":true,\"regression_tests\":{\"duplicate_sha_string_rejected\":true,\"passed\":true,\"recursive_command_identity\":true,\"unique_sha_string_extraction\":true},\"rollback_restored\":null,\"rollback_started\":false,\"root_cause_classification\":\"EXACT_PRIOR_GITEA_DEEP_ANALYSIS_OUTPUT_PROJECTED\",\"schema\":1,\"status\":\"OK\"}\n" +} diff --git a/runtime/history/HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z.txt b/runtime/history/HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z.txt new file mode 100644 index 0000000..e3f74ea --- /dev/null +++ b/runtime/history/HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z.txt @@ -0,0 +1,34 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z +STATUS=OK +RC=0 +HOST=pve01 +MODE=verify +COMPONENT=cluster-knowledge-base-error-system +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83 +COMMAND_SHA256=8d626b3d942e0c8ad8135e1fbf4148a82c5df736a52cc969eb0f6ae022f1ee59 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGE_DECLARED=false +RESULT_CONTRACT_VALID=true +RESULT_CONTRACT_STATUS=NOT_APPLICABLE +RESULT_CONTRACT_ERROR=NONE +COMMAND_RC=0 +CHANGES_MADE=false +ROLLBACK_STARTED=false +ROLLBACK_RESTORED=null +MUTATION_OUTCOME=NO_MUTATION +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=2a7ec04b2e7036e6cc5751216e516b7d78ef56dc135326379580ef58dbc66a29 +SANITIZED_OUTPUT_SHA256=2a7ec04b2e7036e6cc5751216e516b7d78ef56dc135326379580ef58dbc66a29 +OUTPUT_BEGIN +HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z","command_rc":0,"mutation_outcome":"NO_MUTATION","output_truncated_in_json":false,"rc":0,"rollback_restored":null,"rollback_started":false,"status":"OK","version":1} +{"bounded_adapter_scan":null,"changes_made":false,"command_id":"HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z","command_rc":0,"evidence_bundle":{"bundle_bytes_ok":true,"bundle_sha256_ok":true,"exists":true,"member_count":6,"output_member_matches":0,"path_is_exact_from_previous_result":true,"runner_json_member_matches":1,"runner_text_member_matches":1,"unsafe_member_rejected":false},"files_written":false,"mutation_outcome":"NO_MUTATION","network_requests_made":false,"next_action":"RUN_ONE_READ_ONLY_GITEA_DEFINED_CLUSTER_FACT_AND_DOCUMENT_GAP_ANALYSIS","output_truncated_in_json":false,"pre_command_known_error_gate":{"checked":true,"count":28,"ids":["candidate-file-limit-600-20260805","runner-wrapper-mismatch-after-success-20260805","canonical-cluster-handbook-ambiguous-20260805","pvesh-adapter-context-unreadable-20260805","homelab-ops-bare-repository-not-found-local-20260805","gitea-network-fallback-stage-not-emitted-20260805","gitea-owner-hardcoded-homelab-admin-20260805","gitea-git-auth-required-or-credential-unavailable-20260805","gitea-process-self-match-payload-name-20260805","sanitize-bool-attributeerror-20260805","reader-v3-path-misclassified-as-route-20260805","reader-v3-loopback-assumption-20260805","reader-v3-capability-not-treated-as-secret-20260805","reader-v3-systemd-unit-fed-to-python-ast-20260805","reader-v3-helper-control-flow-not-traced-20260805","reader-v3-post-capability-operation-not-derived-20260805","reader-v3-detected-operations-not-tested-20260805","reader-v3-manifest-global-not-bound-in-sandbox-20260805","reader-v3-result-file-second-source-not-read-20260805","reader-v3-negative-control-mixed-with-production-20260805","reader-v3-manifest-schema-guessed-20260805","reader-result-output-bound-after-compaction-20260805","reader-v3-generation-record-shape-guessed-20260805","reader-v3-result-file-assumed-standalone-json-20260805","reader-v3-gitea-detail-marker-false-inventory-20260805","reverse-proxy-runtime-not-visible-on-host-20260805","gitea-public-owner-not-anonymously-discoverable-20260805","exact-output-long-line-not-projectable-by-portal-view-20260805"]},"projection":{"analysis_outcome":{"error_fingerprint":"26d65952ac8f64197a6fd76ec56e3b5ffda8c4838efe2af2947a603d01270b92","next_action":"RUN_ONE_READ_ONLY_GITEA_DEFINED_CLUSTER_FACT_AND_DOCUMENT_GAP_ANALYSIS","ready_for_gitea_defined_cluster_fact_analysis":true,"root_cause_classification":"GITEA_CLUSTER_HANDBOOK_AND_ERROR_SYSTEM_DEEPLY_ANALYZED","status":"OK"},"canonical_handbook":{"candidates":[{"canonical_phrase_hits":3,"canonical_score":227,"content_sha256":"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64","headings":[],"inbound_reference_count":8,"path":"errors/regression-cases.json"},{"canonical_phrase_hits":0,"canonical_score":74,"content_sha256":"a49bcb32ac7212d7fb9768db8a277cb5525c873c84dd6fb633edae35dc6de532","headings":[],"inbound_reference_count":0,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh"},{"canonical_phrase_hits":1,"canonical_score":56,"content_sha256":"ac5dc881dcd9fcb72644bc893e866f463c9c2b9b4544e32434e21b5d34168fb5","headings":[],"inbound_reference_count":1,"path":"inventory/access-paths.json"},{"canonical_phrase_hits":0,"canonical_score":55,"content_sha256":"666072447ba11d346772f62f274dadc974c4463e20f67e67f82f228f1fcf2ce8","headings":[],"inbound_reference_count":0,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/rollback.sh"},{"canonical_phrase_hits":0,"canonical_score":54,"content_sha256":"636254d2526f0f62184d47cdb4c30c2831b11869c2000c557666f60b55d7691b","headings":[],"inbound_reference_count":0,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/task.json"}],"path":"errors/regression-cases.json","status":"UNIQUE_CONFIRMED"},"documentation_coverage":{"access":{"file_count":22,"paths":["changes/CR-2026-0018/regression-cases-draft.json","docs/planned/skladchik-moderator-assistant.md","errors/regression-cases.json","inventory/access-paths.json","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/dry-run.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/prepare.sh"]},"backup":{"file_count":29,"paths":["docs/ARCHITECTURE.md","docs/skladchik-cookie-refresh.md","errors/regression-cases.json","observed/cluster-guests.json","policies/cluster-cloud-quorum.json","schemas/change.schema.json","tasks/cluster-cloud-quorum-vm160-canary-v1/assets/pve02-vm160-preflight-probe.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/assets/vm160-cloud-quorum-worker.sh"]},"error_system":{"file_count":44,"paths":[".gitea/workflows/ci.yml","changes/CR-2026-0018/caller-inventory-requirements.md","changes/CR-2026-0018/regression-cases-draft.json","changes/README.md","docs/ARCHITECTURE.md","docs/skladchik-cookie-refresh.md","docs/WORKFLOW.md","errors/regression-cases.json"]},"internal_external":{"file_count":7,"paths":["changes/CR-2026-0018/regression-cases-draft.json","errors/regression-cases.json","policies/operating-policy.json","tasks/homelab-logrotate-namespace-latch-clearance-v1/README.txt","tests/test_cr_2026_0005.py","tests/test_cr_2026_0011_skladchik_cookie_jar_reset.py","tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py"]},"repair":{"file_count":31,"paths":["changes/CR-2026-0018/caller-inventory-requirements.md","changes/README.md","docs/ARCHITECTURE.md","docs/skladchik-cookie-refresh.md","docs/WORKFLOW.md","errors/regression-cases.json","policies/cluster-cloud-quorum.json","policies/operating-policy.json"]},"services":{"file_count":21,"paths":["changes/CR-2026-0018/caller-inventory-requirements.md","docs/ARCHITECTURE.md","docs/planned/skladchik-moderator-assistant.md","errors/regression-cases.json","observed/cluster-guests.json","tasks/cluster-cloud-quorum-vm160-canary-v1/assets/vm160-cloud-quorum-worker.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/dry-run.sh"]}},"enumeration":{"candidate_file_count":50,"fetch_failure_count":0,"fetched_file_count":50,"fetched_total_bytes":173536,"tree_blob_count":125,"tree_complete":true,"tree_directory_count":36,"tree_entry_count":161},"error_system_current":{"duplicate_error_ids":[],"duplicate_fingerprints":[],"enforcement_path_count":7,"enforcement_paths":["schemas/change.schema.json","schemas/context.schema.json","tasks/cluster-cloud-quorum-vm160-canary-v1/assets/pve02-vm160-preflight-probe.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh","tasks/homelab-logrotate-namespace-latch-clearance-v1/phases/preflight.sh","tasks/skladchik-reports-monitor-controlled-reauth-v1/assets/edge-preflight.sh","tasks/skladchik-reports-monitor-controlled-reauth-v1/phases/preflight.sh"],"gate_signals":{"ci_workflow_present":true,"duplicate_failed_command_block_signal":false,"error_register_checked_signal":false,"fingerprint_signal":false,"preflight_references_error_register":true,"reference_register_checked_signal":false,"schema_validation_signal":true,"tests_present":true},"machine_parseable_registry_file_count":2,"prevention_evidence_count":0,"registry_path_count":2,"registry_paths":["changes/CR-2026-0018/regression-cases-draft.json","errors/regression-cases.json"],"regression_path_count":2,"regression_paths":["changes/CR-2026-0018/regression-cases-draft.json","errors/regression-cases.json"],"superseded_path_count":0,"superseded_paths":[],"test_path_count":14,"test_paths":["tests/fixtures/gitea-legacy-auth-permission.json","tests/test_cr_2026_0005.py","tests/test_cr_2026_0006.py","tests/test_cr_2026_0006_vm160_task.py","tests/test_cr_2026_0007_skladchik_reauth.py","tests/test_cr_2026_0007_skladchik_semantic_repair.py","tests/test_cr_2026_0008_skladchik_snapshot_integrity.py","tests/test_cr_2026_0009_skladchik_concurrency_guard.py","tests/test_cr_2026_0011_skladchik_cookie_jar_reset.py","tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py"],"unique_error_id_count":61,"unique_fingerprint_count":0,"workflow_path_count":1,"workflow_paths":[".gitea/workflows/ci.yml"]},"gaps":["no_static_evidence_error_fingerprint_gate"],"pre_command_gate":{"checked":true,"projection_command_known_error_count":28,"regression_tests_passed":true,"reported_error_count":27},"previous_exact_result":{"capsule_sha256":"031aa6c1029c24d8b7dac8192707376fa67ee93cdcfceedcfb768a0b7d4ceb8e","command_id":"HOMELAB-CLUSTER-KB-V1-GITEA-REPOSITORY-CONTENT-DEEP-ANALYSIS-READ-ONLY-20260805T124500Z","identity_fully_verified":true,"output_bytes":27219,"output_sha256":"ead28fd156e646c27f6e988c12ea1e89a498882f07071faacba670e87c8a7983","runner_json_sha256":"e3211267d27f7497339cd34ecdbe0cce21d039965625ffd5f3236dc20513b082","runner_text_sha256":"083286377044254ff016f0f9024655e228a67ae2737d7e1d9641996d5af8e09a"},"previous_result_contract":{"changes_made":false,"command_rc":0,"mutation_outcome":"NO_MUTATION","output_truncated_in_json":false,"rc":0,"rollback_restored":null,"rollback_started":false,"status":"OK"},"recommended_error_system_v2":{"acceptance_criteria":["zero hard-coded owner/path/host/version unless proven by exact prior evidence or current Gitea content","zero mutation without byte-for-byte backup and tested rollback","zero repeated known error fingerprint","all command outputs bounded and non-truncated","all new errors recorded before any subsequent command"],"components":[{"component":"machine_error_registry","requirement":"one append-only machine-readable record per proven failure with stable id, normalized fingerprint, exact command/capsule/result SHA, root cause, correction and prevention test"},{"component":"pre_command_gate","requirement":"before every .hcmd, read canonical cluster reference and entire active error registry; block duplicate failed fingerprints and commands that violate a prevention test"},{"component":"transaction_manifest","requirement":"every mutation declares exact source SHA, candidate SHA, byte backup SHA, preconditions, atomic apply, verification and deterministic rollback"},{"component":"post_result_ingestion","requirement":"after exact-result verification, automatically classify the outcome, append new proven failures, close corrected cases and link superseded commands"},{"component":"regression_suite","requirement":"positive and negative fixtures for every recorded error; payload self-test and repository CI must pass before command publication"},{"component":"knowledge_freshness","requirement":"canonical handbook stores last verified fact source, verification time, source SHA and stale/unknown markers; commands may not rely on stale or unknown values"}],"grounded_in_detected_gaps":["no_static_evidence_error_fingerprint_gate"],"minimum_gate_order":["verify previous exact result identity","read canonical handbook at exact branch commit","read active errors and prevention tests","validate target facts from authoritative source","run payload regression tests","only then publish or execute the next command"],"name":"Error System V2: evidence-first command gate","reuse_existing_error_root":"changes/CR-2026-0018"},"repository":{"branch_commit_sha":"fe1221b06643db3b00a621b0230f92a46a4edc2b","default_branch":"main","full_name":"homelab-admin/homelab-ops","private":true},"source_result_projection":{"budget_bytes":29232,"bytes":26771,"max_bytes":30000,"steps":["top_analyzed_files_12","handbook_candidates_6"]}},"projection_bytes_before_final_wrapper":11730,"rc":0,"read_only":true,"regression_tests":{"duplicate_sha_string_rejected":true,"passed":true,"recursive_command_identity":true,"unique_sha_string_extraction":true},"rollback_restored":null,"rollback_started":false,"root_cause_classification":"EXACT_PRIOR_GITEA_DEEP_ANALYSIS_OUTPUT_PROJECTED","schema":1,"status":"OK"} + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 119b1d1..e399a6c 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z", + "command_id": "HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z", "status": "OK", "rc": 0, "host": "pve01", "mode": "verify", - "component": "homelab-cluster-admin-update-visibility", - "started_at_utc": "2026-08-05T12:55:27Z", - "finished_at_utc": "2026-08-05T12:55:29Z", + "component": "cluster-knowledge-base-error-system", + "started_at_utc": "2026-08-05T13:08:01Z", + "finished_at_utc": "2026-08-05T13:08:01Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83", - "command_sha256": "4845b4e99e60efaa533b2a551d4bf1773085ce96ef0e4ce9737b920907b0553f", + "command_sha256": "8d626b3d942e0c8ad8135e1fbf4148a82c5df736a52cc969eb0f6ae022f1ee59", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -30,9 +30,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "b672b2fde18de662e73d8a98d4cc036895f9de2007679edc3c085392050e66e6", - "sanitized_output_sha256": "c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8", + "raw_evidence_sha256": "2a7ec04b2e7036e6cc5751216e516b7d78ef56dc135326379580ef58dbc66a29", + "sanitized_output_sha256": "2a7ec04b2e7036e6cc5751216e516b7d78ef56dc135326379580ef58dbc66a29", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z\",\"command_rc\":0,\"control_plane_error\":null,\"mutation_outcome\":\"NO_MUTATION\",\"rc\":0,\"ready_for_corrected_atomic_transaction\":true,\"remote_application_candidate_count\":40,\"remote_hostname\":\"cluster-admin\",\"remote_index_count\":1,\"remote_probe_privilege\":\"sudo-root\",\"remote_reference_file_count\":1,\"remote_target_cardinality\":1,\"remote_target_host_sha256\":\"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69\",\"remote_unit_count\":16,\"remote_webroot_config_count\":0,\"report_bytes\":85056,\"report_sha256\":\"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf\",\"rollback_restored\":false,\"rollback_started\":false,\"status\":\"OK\",\"v14_exact_result_identity_confirmed\":true,\"version\":1,\"webpanel_script_sha256\":\"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276\"}\n{\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z\",\"mutation_outcome\":\"NO_MUTATION\",\"read_only\":true,\"remote_architecture\":{\"application_candidates\":[{\"bytes\":133,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},{\"bytes\":149,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},{\"bytes\":151,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},{\"bytes\":231,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},{\"bytes\":138,\"matches\":[{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},{\"bytes\":127,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},{\"bytes\":616,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},{\"bytes\":1190,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},{\"bytes\":223,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},{\"bytes\":803,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},{\"bytes\":131,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},{\"bytes\":301,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},{\"bytes\":89,\"matches\":[{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\"},{\"bytes\":8444,\"matches\":[{\"line\":4,\"text\":\"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')\"},{\"line\":64,\"text\":\" return f\\\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\\\n\\\"\"},{\"line\":82,\"text\":\" old=\"},{\"line\":97,\"text\":\" p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)\"},{\"line\":99,\"text\":\" STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\"},{\"bytes\":28599,\"matches\":[{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =\"},{\"line\":14,\"text\":\"COOKIE =\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":20,\"text\":\" return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":73,\"text\":\" return f\\\"\\\"\\\"\"},{\"line\":152,\"text\":\" return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":220,\"text\":\" page = f\\\"\\\"\\\"Cluster Admin Incident Engine
Cluster Admin Incident Engine {engine_badge}
observe/notify · auto-heal отключён
{cards}
{nodes_html}{incidents_html}{services_html}
\\\"\\\"\\\"\"},{\"line\":243,\"text\":\" \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\" \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\" \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\" \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\" \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\" \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\" \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\" \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\" \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\" \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\" \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\" \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\" \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\" \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\" \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\" \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\" \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\" \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\" \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\" \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\" \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":267,\"text\":\" lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\"},{\"bytes\":922,\"matches\":[{\"line\":2,\"text\":\"first = getpass.getpass(\\\"Новый пароль Cluster Admin: \\\")\"},{\"line\":12,\"text\":\"directory = \\\"/etc/cluster-admin-incident-engine\\\"\"},{\"line\":16,\"text\":\"account = pwd.getpwnam(\\\"clusteradmin\\\")\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\"},{\"bytes\":10640,\"matches\":[{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":36,\"text\":\" data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\" connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":41,\"text\":\" cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":50,\"text\":\" if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":57,\"text\":\" cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":108,\"text\":\" line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\"},{\"bytes\":1857,\"matches\":[{\"line\":3,\"text\":\" \\\"type\\\": \\\"cluster-admin-stage1-foundation-migration-plan\\\",\"},{\"line\":43,\"text\":\" \\\"function\\\": \\\"cluster_admin_enrich_collector_run\\\",\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\"},{\"bytes\":1342,\"matches\":[{\"line\":3,\"text\":\" \\\"collector_call\\\": \\\"SELECT cluster_admin_sync_foundation(%s)\\\",\"},{\"line\":15,\"text\":\" \\\"function\\\": \\\"cluster_admin_sync_foundation\\\",\"},{\"line\":16,\"text\":\" \\\"function_signature\\\": \\\"cluster_admin_sync_foundation(bigint)\\\",\"},{\"line\":26,\"text\":\" \\\"type\\\": \\\"cluster-admin-stage2-migration-plan\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\"},{\"bytes\":2355,\"matches\":[{\"line\":51,\"text\":\" \\\"type\\\": \\\"cluster-admin-stage2-dynamic-acceptance-contract\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\"},{\"bytes\":2832,\"matches\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\" line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\" echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"Homelab Cluster Admin\"},{\"line\":58,\"text\":\"

Homelab Cluster Admin

\"},{\"line\":74,\"text\":\"

Service map tail

$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)
\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},{\"bytes\":3471,\"matches\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\" echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\" echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},{\"bytes\":1433,\"matches\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},{\"bytes\":848,\"matches\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":10353,\"matches\":[{\"line\":7,\"text\":\"Homelab Cluster Admin\"},{\"line\":19,\"text\":\"

Homelab Cluster Admin

\"},{\"line\":24,\"text\":\"

Self

STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"
Observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
\"},{\"line\":28,\"text\":\"
Full observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
\"},{\"line\":29,\"text\":\"
Mail Cloud backup
STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md
\"},{\"line\":32,\"text\":\"
\"},{\"line\":33,\"text\":\"
Final readiness
STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md
\"},{\"line\":34,\"text\":\"
Strict seal
STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md
\"},{\"line\":35,\"text\":\"
Service map tail
# Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":46,\"text\":\"OK severity=\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\"},{\"bytes\":168,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729\"},{\"bytes\":259,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741\"},{\"bytes\":134,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517\"},{\"bytes\":141,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\"},{\"bytes\":189,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\"},{\"bytes\":131,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":199,\"matches\":[{\"line\":1,\"text\":\"STATUS=\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\"},{\"bytes\":272,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec\"},{\"bytes\":380,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"}],\"euid\":0,\"gate\":{\"application_candidate_count\":40,\"ready_for_remote_transaction_design\":true,\"reference_file_count\":1,\"unit_count\":16},\"health_input_files\":[{\"bytes\":380,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":217,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"},{\"bytes\":128,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\"},{\"bytes\":1535,\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\"}],\"hostname\":\"cluster-admin\",\"indexes\":[{\"bytes\":10353,\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"root\":\"/var/www/homelab-cluster-admin\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\",\"title_lines\":[\"Homelab Cluster Admin\",\"
Homelab Cluster Admin
\",\"
Self
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\",\"
Observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
\",\"
Full observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
\",\"
Mail Cloud backup
STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md
\",\"
\",\"
Final readiness
STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md
\",\"
Strict seal
STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md
\",\"
Service map tail
# Homelab cluster-admin full observer\",\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\",\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"]}],\"listeners\":{\"rc\":0,\"stderr\":\"\",\"stdout\":\"State  Recv-Q Send-Q Local Address:Port Peer Address:PortProcess                                   \\nLISTEN 0      5            0.0.0.0:8080      0.0.0.0:*    users:((\\\"python3\\\",pid=439,fd=3))         \\nLISTEN 0      2048         0.0.0.0:8081      0.0.0.0:*    users:((\\\"python3\\\",pid=487,fd=6))         \\nLISTEN 0      244        127.0.0.1:5432      0.0.0.0:*    users:((\\\"postgres\\\",pid=469,fd=6))        \\nLISTEN 0      4096   127.0.0.53%lo:53        0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=18))\\nLISTEN 0      4096         0.0.0.0:5355      0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=12))\\nLISTEN 0      4096      127.0.0.54:53        0.0.0.0:*    users:((\\\"systemd-resolve\\\",pid=380,fd=20))\\nLISTEN 0      128          0.0.0.0:22        0.0.0.0:*    users:((\\\"sshd\\\",pid=457,fd=3))            \\nLISTEN 0      244            [::1]:5432         [::]:*    users:((\\\"postgres\\\",pid=469,fd=5))        \\nLISTEN 0      4096            [::]:5355         [::]:*    users:((\\\"systemd-resolve\\\",pid=380,fd=14))\\nLISTEN 0      128             [::]:22           [::]:*    users:((\\\"sshd\\\",pid=457,fd=4))            \"},\"matching_files\":[{\"bytes\":133,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin full observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/full-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"sha256\":\"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b\"},{\"bytes\":149,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"sha256\":\"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Refresh Homelab cluster-admin web panel\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"sha256\":\"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311\"},{\"bytes\":151,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin self check\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"sha256\":\"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1\"},{\"bytes\":231,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin incident collector every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-incident-engine-collector.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"sha256\":\"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae\"},{\"bytes\":154,\"matches\":[{\"line\":2,\"text\":\"Description=Run homelab cluster-admin full observer\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"sha256\":\"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23\"},{\"bytes\":138,\"matches\":[{\"line\":2,\"text\":\"Description=Render Homelab cluster-admin web panel\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"sha256\":\"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e\"},{\"bytes\":127,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin self check\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/self-check.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"sha256\":\"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d\"},{\"bytes\":616,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin inventory metrics and incident collector\"},{\"line\":3,\"text\":\"After=network-online.target postgresql.service cluster-admin-incident-engine.service\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\"},{\"line\":18,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":19,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"sha256\":\"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7\"},{\"bytes\":1190,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin restricted read-only probe controller\"},{\"line\":8,\"text\":\"User=clusteradmin\"},{\"line\":9,\"text\":\"Group=clusteradmin\"},{\"line\":10,\"text\":\"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\"},{\"line\":11,\"text\":\"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\"},{\"line\":37,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\"},{\"line\":38,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-probe-agent\"},{\"line\":39,\"text\":\"StateDirectory=cluster-admin-probe-agent\"},{\"line\":41,\"text\":\"RuntimeDirectory=cluster-admin-probe-agent\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"sha256\":\"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771\"},{\"bytes\":223,\"matches\":[{\"line\":2,\"text\":\"Description=Run Cluster Admin restricted probes every minute\"},{\"line\":9,\"text\":\"Unit=cluster-admin-probe-agent.service\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"sha256\":\"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d\"},{\"bytes\":803,\"matches\":[{\"line\":2,\"text\":\"Description=Cluster Admin Incident Engine web application\"},{\"line\":9,\"text\":\"User=clusteradmin\"},{\"line\":10,\"text\":\"Group=clusteradmin\"},{\"line\":11,\"text\":\"WorkingDirectory=/opt/cluster-admin-incident-engine\"},{\"line\":12,\"text\":\"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\"},{\"line\":21,\"text\":\"ReadOnlyPaths=/etc/cluster-admin-incident-engine\"},{\"line\":22,\"text\":\"ReadWritePaths=/var/lib/cluster-admin-incident-engine\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"sha256\":\"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea\"},{\"bytes\":131,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin observer\"},{\"line\":6,\"text\":\"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"sha256\":\"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042\"},{\"bytes\":301,\"matches\":[{\"line\":2,\"text\":\"Description=Homelab cluster-admin web panel\"},{\"line\":8,\"text\":\"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"sha256\":\"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b\"},{\"bytes\":89,\"matches\":[{\"line\":3,\"text\":\"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"}],\"mode\":\"0o644\",\"path\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"sha256\":\"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f\"},{\"bytes\":8444,\"matches\":[{\"line\":4,\"text\":\"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')\"},{\"line\":64,\"text\":\" return f\\\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\\\n\\\"\"},{\"line\":82,\"text\":\" old=\"},{\"line\":97,\"text\":\"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)\"},{\"line\":99,\"text\":\"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-probe-agent/controller.py\",\"sha256\":\"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a\"},{\"bytes\":28599,\"matches\":[{\"line\":12,\"text\":\"KEY = open(\\\"/etc/cluster-admin-incident-engine/web.key\\\", \\\"rb\\\").read().strip()\"},{\"line\":13,\"text\":\"PASSFILE =\"},{\"line\":14,\"text\":\"COOKIE =\"},{\"line\":17,\"text\":\"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\\\"pvepro.ru\\\",\\\"www.pvepro.ru\\\",\\\"cluster-admin.vpn.gram1.ru\\\",\\\"[PRIVATE_IP]\\\",\\\"127.0.0.1\\\",\\\"localhost\\\"])\"},{\"line\":20,\"text\":\"    return psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":73,\"text\":\"    return f\\\"\\\"\\\"\"},{\"line\":152,\"text\":\"    return {\\\"status\\\":\\\"OK\\\",\\\"service\\\":\\\"cluster-admin-incident-engine\\\",\\\"mode\\\":\\\"observe-notify\\\"}\"},{\"line\":220,\"text\":\"    page = f\\\"\\\"\\\"Cluster Admin Incident Engine
Cluster Admin Incident Engine {engine_badge}
observe/notify · auto-heal отключён
{cards}
{nodes_html}{incidents_html}{services_html}
\\\"\\\"\\\"\"},{\"line\":243,\"text\":\"        \\\"# HELP cluster_admin_up Cluster Admin Incident Engine availability\\\",\"},{\"line\":244,\"text\":\"        \\\"# TYPE cluster_admin_up gauge\\\",\"},{\"line\":245,\"text\":\"        \\\"cluster_admin_up 1\\\",\"},{\"line\":246,\"text\":\"        \\\"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\\\",\"},{\"line\":247,\"text\":\"        \\\"# TYPE cluster_admin_nodes_total gauge\\\",\"},{\"line\":248,\"text\":\"        \\\"cluster_admin_nodes_total \\\" + str(nodes),\"},{\"line\":249,\"text\":\"        \\\"# HELP cluster_admin_guests_total Discovered virtual machines and containers\\\",\"},{\"line\":250,\"text\":\"        \\\"# TYPE cluster_admin_guests_total gauge\\\",\"},{\"line\":251,\"text\":\"        \\\"cluster_admin_guests_total \\\" + str(guests),\"},{\"line\":252,\"text\":\"        \\\"# HELP cluster_admin_services_total Discovered services\\\",\"},{\"line\":253,\"text\":\"        \\\"# TYPE cluster_admin_services_total gauge\\\",\"},{\"line\":254,\"text\":\"        \\\"cluster_admin_services_total \\\" + str(services),\"},{\"line\":255,\"text\":\"        \\\"# HELP cluster_admin_observations_total Stored metric observations\\\",\"},{\"line\":256,\"text\":\"        \\\"# TYPE cluster_admin_observations_total gauge\\\",\"},{\"line\":257,\"text\":\"        \\\"cluster_admin_observations_total \\\" + str(observations),\"},{\"line\":258,\"text\":\"        \\\"# HELP cluster_admin_incidents_active_total Active correlated incidents\\\",\"},{\"line\":259,\"text\":\"        \\\"# TYPE cluster_admin_incidents_active_total gauge\\\",\"},{\"line\":260,\"text\":\"        \\\"cluster_admin_incidents_active_total \\\" + str(incidents),\"},{\"line\":261,\"text\":\"        \\\"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\\\",\"},{\"line\":262,\"text\":\"        \\\"# TYPE cluster_admin_collector_age_seconds gauge\\\",\"},{\"line\":263,\"text\":\"        \\\"cluster_admin_collector_age_seconds \\\" + str(collector_age)\"},{\"line\":267,\"text\":\"        lines.append(\\\"cluster_admin_incidents_active{severity=\\\" + chr(34) + clean + chr(34) + \\\"} \\\" + str(count))\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/app.py\",\"sha256\":\"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6\"},{\"bytes\":922,\"matches\":[{\"line\":2,\"text\":\"first = getpass.getpass(\\\"Новый пароль Cluster Admin: \\\")\"},{\"line\":12,\"text\":\"directory = \\\"/etc/cluster-admin-incident-engine\\\"\"},{\"line\":16,\"text\":\"account = pwd.getpwnam(\\\"clusteradmin\\\")\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/set-password.py\",\"sha256\":\"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3\"},{\"bytes\":10640,\"matches\":[{\"line\":9,\"text\":\"INVENTORY = \\\"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\\\"\"},{\"line\":10,\"text\":\"HEALTH = \\\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\\\"\"},{\"line\":12,\"text\":\"HOSTS = {\\\"[PRIVATE_IP]\\\":\\\"pve01\\\",\\\"[PRIVATE_IP]\\\":\\\"pve02\\\",\\\"[PRIVATE_IP]\\\":\\\"pve03\\\",\\\"[PRIVATE_IP]\\\":\\\"edge-vm\\\",\\\"[PRIVATE_IP]\\\":\\\"nextcloud\\\",\\\"[PRIVATE_IP]\\\":\\\"forum-prod\\\",\\\"[PRIVATE_IP]\\\":\\\"core-apps\\\",\\\"[PRIVATE_IP]\\\":\\\"monitoring\\\",\\\"[PRIVATE_IP]\\\":\\\"cluster-admin\\\"}\"},{\"line\":36,\"text\":\"    data = json.load(open(INVENTORY, encoding=\\\"utf-8\\\"))\"},{\"line\":37,\"text\":\"    connection = psycopg2.connect(dbname=\\\"clusteradmin\\\", user=\\\"clusteradmin\\\", host=\\\"/var/run/postgresql\\\")\"},{\"line\":41,\"text\":\"        cursor.execute(\\\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\\\",(\\\"cluster-inventory-prometheus\\\",\\\"RUNNING\\\",Json({\\\"inventory_generated_at\\\":data.get(\\\"generated_at\\\")})))\"},{\"line\":50,\"text\":\"            if str(item.get(\\\"name\\\", \\\"\\\")) != \\\"cluster-admin-incident-engine\\\" and str(item.get(\\\"status\\\", \\\"\\\")).upper() != \\\"OK\\\":\"},{\"line\":57,\"text\":\"        cursor.execute(\\\"SELECT cluster_admin_sync_foundation(%s)\\\",(run_id,))\"},{\"line\":108,\"text\":\"    line = \\\"STATUS=OK TS=\\\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \\\" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\\\" + str(len(data.get(\\\"nodes\\\",[]))) + \\\" GUESTS=\\\" + str(len(data.get(\\\"guests\\\",[]))) + \\\" SERVICES=\\\" + str(len(data.get(\\\"services\\\",[]))) + \\\" METRICS=\\\" + str(metric_count) + \\\" OPEN_INCIDENTS=\\\" + str(open_count) + \\\" MODE=observe-notify\\\\n\\\"\"}],\"mode\":\"0o750\",\"path\":\"/opt/cluster-admin-incident-engine/collector.py\",\"sha256\":\"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe\"},{\"bytes\":1857,\"matches\":[{\"line\":3,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage1-foundation-migration-plan\\\",\"},{\"line\":43,\"text\":\"    \\\"function\\\": \\\"cluster_admin_enrich_collector_run\\\",\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json\",\"sha256\":\"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670\"},{\"bytes\":1342,\"matches\":[{\"line\":3,\"text\":\"  \\\"collector_call\\\": \\\"SELECT cluster_admin_sync_foundation(%s)\\\",\"},{\"line\":15,\"text\":\"  \\\"function\\\": \\\"cluster_admin_sync_foundation\\\",\"},{\"line\":16,\"text\":\"  \\\"function_signature\\\": \\\"cluster_admin_sync_foundation(bigint)\\\",\"},{\"line\":26,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-migration-plan\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json\",\"sha256\":\"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c\"},{\"bytes\":2355,\"matches\":[{\"line\":51,\"text\":\"  \\\"type\\\": \\\"cluster-admin-stage2-dynamic-acceptance-contract\\\"\"}],\"mode\":\"0o644\",\"path\":\"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json\",\"sha256\":\"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588\"},{\"bytes\":2832,\"matches\":[{\"line\":3,\"text\":\"LOCK=/run/homelab-cluster-admin-observer.lock\"},{\"line\":5,\"text\":\"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":8,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":9,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":11,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-observer.txt\\\"\"},{\"line\":35,\"text\":\"  line=$(curl -fsS --max-time 5 \\\"$url\\\" 2>/tmp/cluster-admin-curl.err | head -n1)\"},{\"line\":45,\"text\":\"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)\"},{\"line\":48,\"text\":\"  echo \\\"# Homelab cluster-admin observer\\\"\"},{\"line\":85,\"text\":\"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt\"},{\"line\":88,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/cluster-observer.sh\",\"sha256\":\"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5\"},{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"Homelab Cluster Admin\"},{\"line\":58,\"text\":\"
Homelab Cluster Admin
\"},{\"line\":74,\"text\":\"
Service map tail
$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)
\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"},{\"bytes\":3471,\"matches\":[{\"line\":4,\"text\":\"ROOT=/var/lib/homelab-cluster-admin\"},{\"line\":5,\"text\":\"HEALTH_DIR=/var/lib/homelab-health\"},{\"line\":6,\"text\":\"MAP=/etc/homelab-cluster-admin/service-map.tsv\"},{\"line\":8,\"text\":\"HEALTH=\\\"$HEALTH_DIR/cluster-admin-full-observer.txt\\\"\"},{\"line\":9,\"text\":\"LOCK=/run/homelab-cluster-admin-full-observer.lock\"},{\"line\":12,\"text\":\"flock -n 9 || { cat \\\"$HEALTH\\\" 2>/dev/null || echo \\\"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\\\"; exit 0; }\"},{\"line\":14,\"text\":\"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519\"},{\"line\":26,\"text\":\"  echo \\\"# Homelab cluster-admin full observer\\\"\"},{\"line\":28,\"text\":\"  echo \\\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\\\"\"},{\"line\":105,\"text\":\"LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\\\"\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/full-observer.sh\",\"sha256\":\"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e\"},{\"bytes\":1433,\"matches\":[{\"line\":3,\"text\":\"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh\"},{\"line\":4,\"text\":\"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt\"},{\"line\":5,\"text\":\"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt\"}],\"mode\":\"0o750\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\",\"sha256\":\"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792\"},{\"bytes\":848,\"matches\":[{\"line\":16,\"text\":\"echo \\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\\\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null\"},{\"line\":17,\"text\":\"cat /var/lib/homelab-health/cluster-admin-self-check.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/self-check.sh\",\"sha256\":\"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/status.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":10353,\"matches\":[{\"line\":7,\"text\":\"Homelab Cluster Admin\"},{\"line\":19,\"text\":\"
Homelab Cluster Admin
\"},{\"line\":24,\"text\":\"
Self
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"},{\"line\":27,\"text\":\"
Observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
\"},{\"line\":28,\"text\":\"
Full observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
\"},{\"line\":29,\"text\":\"
Mail Cloud backup
STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md
\"},{\"line\":32,\"text\":\"
\"},{\"line\":33,\"text\":\"
Final readiness
STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md
\"},{\"line\":34,\"text\":\"
Strict seal
STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md
\"},{\"line\":35,\"text\":\"
Service map tail
# Homelab cluster-admin full observer\"},{\"line\":37,\"text\":\"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\"},{\"line\":40,\"text\":\"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2\"},{\"line\":41,\"text\":\"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2\"},{\"line\":42,\"text\":\"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0\"},{\"line\":43,\"text\":\"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active\"},{\"line\":46,\"text\":\"OK severity=\"},{\"line\":57,\"text\":\"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0\"},{\"line\":78,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
\"}],\"mode\":\"0o644\",\"path\":\"/var/www/homelab-cluster-admin/index.html\",\"sha256\":\"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a\"},{\"bytes\":168,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\",\"sha256\":\"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729\"},{\"bytes\":259,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-full-observer.txt\",\"sha256\":\"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741\"},{\"bytes\":134,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-self-check.txt\",\"sha256\":\"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517\"},{\"bytes\":141,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt\",\"sha256\":\"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6\"},{\"bytes\":189,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt\",\"sha256\":\"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941\"},{\"bytes\":131,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0\"}],\"mode\":\"0o640\",\"path\":\"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\",\"sha256\":\"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6\"},{\"bytes\":214,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-webpanel.txt\",\"sha256\":\"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8\"},{\"bytes\":199,\"matches\":[{\"line\":1,\"text\":\"STATUS=\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt\",\"sha256\":\"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680\"},{\"bytes\":272,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/cluster-admin-observer.txt\",\"sha256\":\"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec\"},{\"bytes\":380,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt\",\"sha256\":\"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e\"},{\"bytes\":1535,\"matches\":[{\"line\":1,\"text\":\"STATUS=\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/overall.txt\",\"sha256\":\"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b\"},{\"bytes\":128,\"matches\":[{\"line\":1,\"text\":\"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/final-readiness.txt\",\"sha256\":\"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac\"},{\"bytes\":217,\"matches\":[{\"line\":1,\"text\":\"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\"}],\"mode\":\"0o644\",\"path\":\"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt\",\"sha256\":\"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053\"}],\"process_lines\":[\"    439       1 root     root     python3         /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\",\"    487       1 cluster+ cluster+ python3         /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\"],\"proxies\":[],\"pve01_pushed_reference_files\":[{\"bytes\":3708,\"matches\":[{\"line\":4,\"text\":\"WEB=/var/www/homelab-cluster-admin\"},{\"line\":13,\"text\":\"SELF=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\\\"\"},{\"line\":14,\"text\":\"OBSERVER=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\\\"\"},{\"line\":15,\"text\":\"FULL=\\\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\\\"\"},{\"line\":16,\"text\":\"BACKUP=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\\\"\"},{\"line\":17,\"text\":\"OVERALL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\\\"\"},{\"line\":18,\"text\":\"FINAL=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\\\"\"},{\"line\":19,\"text\":\"STRICT=\\\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\\\"\"},{\"line\":30,\"text\":\"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates\"},{\"line\":35,\"text\":\"STATUS_LINE=\\\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\\\"\"},{\"line\":36,\"text\":\"echo \\\"$STATUS_LINE\\\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null\"},{\"line\":46,\"text\":\"Homelab Cluster Admin\"},{\"line\":58,\"text\":\"
Homelab Cluster Admin
\"},{\"line\":74,\"text\":\"
Service map tail
$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)
\"},{\"line\":79,\"text\":\"cat /var/lib/homelab-health/cluster-admin-webpanel.txt\"}],\"mode\":\"0o755\",\"path\":\"/opt/homelab-cluster-admin/render-webpanel.sh\",\"sha256\":\"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f\"}],\"scan\":{\"directories_scanned\":68,\"index_count\":1,\"match_count\":42},\"units\":[{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-incident-engine-collector.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\\n[Unit]\\nDescription=Cluster Admin inventory metrics and incident collector\\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine-collector.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-incident-engine-collector.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine-collector.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\\n[Unit]\\nDescription=Run Cluster Admin incident collector every minute\\n\\n[Timer]\\nOnBootSec=30s\\nOnUnitActiveSec=60s\\nAccuracySec=10s\\nPersistent=true\\nUnit=cluster-admin-incident-engine-collector.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"EnvironmentFiles\":\"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-incident-engine.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"/opt/cluster-admin-incident-engine\"},\"show_rc\":0,\"unit\":\"cluster-admin-incident-engine.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-incident-engine.service\\n[Unit]\\nDescription=Cluster Admin Incident Engine web application\\nAfter=network-online.target postgresql.service\\nWants=network-online.target\\nRequires=postgresql.service\\n\\n[Service]\\nType=simple\\nUser=clusteradmin\\nGroup=clusteradmin\\nWorkingDirectory=/opt/cluster-admin-incident-engine\\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\\nRestart=on-failure\\nRestartSec=5\\nUMask=0027\\nNoNewPrivileges=true\\nPrivateTmp=true\\nProtectSystem=strict\\nProtectHome=true\\nReadOnlyPaths=/etc/cluster-admin-incident-engine\\nReadWritePaths=/var/lib/cluster-admin-incident-engine\\nRestrictSUIDSGID=true\\nLockPersonality=true\\nRestrictRealtime=true\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.service\",\"Group\":\"clusteradmin\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"cluster-admin-probe-agent.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"clusteradmin\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.service\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.service\\n[Unit]\\nDescription=Cluster Admin restricted read-only probe controller\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=oneshot\\nUser=clusteradmin\\nGroup=clusteradmin\\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\\nUMask=0077\\nNoNewPrivileges=yes\\nPrivateTmp=yes\\nPrivateDevices=yes\\nProtectSystem=strict\\nProtectHome=yes\\nProtectKernelTunables=yes\\nProtectKernelModules=yes\\nProtectKernelLogs=yes\\nProtectControlGroups=yes\\nProtectClock=yes\\nProtectHostname=yes\\nLockPersonality=yes\\nMemoryDenyWriteExecute=yes\\nRestrictSUIDSGID=yes\\nRestrictRealtime=yes\\nRemoveIPC=yes\\nCapabilityBoundingSet=\\nAmbientCapabilities=\\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\\nIPAddressDeny=any\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nIPAddressAllow=[PRIVATE_IP]\\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\\nReadWritePaths=/var/lib/cluster-admin-probe-agent\\nStateDirectory=cluster-admin-probe-agent\\nStateDirectoryMode=0750\\nRuntimeDirectory=cluster-admin-probe-agent\\nTimeoutStartSec=70\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/cluster-admin-probe-agent.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"cluster-admin-probe-agent.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"cluster-admin-probe-agent.timer\",\"unit_text\":\"# /etc/systemd/system/cluster-admin-probe-agent.timer\\n[Unit]\\nDescription=Run Cluster Admin restricted probes every minute\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=60s\\nRandomizedDelaySec=5s\\nPersistent=true\\nUnit=cluster-admin-probe-agent.service\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-full-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin full observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/full-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-full-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-full-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-full-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin full observer\\n\\n[Timer]\\nOnBootSec=5min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-observer.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.service\\n[Unit]\\nDescription=Homelab cluster-admin observer\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-observer.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-observer.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-observer.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-observer.timer\\n[Unit]\\nDescription=Run homelab cluster-admin observer\\n\\n[Timer]\\nOnBootSec=3min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-self-check.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.service\\n[Unit]\\nDescription=Homelab cluster-admin self check\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/self-check.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-self-check.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-self-check.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-self-check.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\\n[Unit]\\nDescription=Run homelab cluster-admin self check\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=5min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"inactive\",\"DropInPaths\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\",\"ExecStart\":\"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"dead\",\"TriggeredBy\":\"homelab-cluster-admin-webpanel-render.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\\n[Unit]\\nDescription=Render Homelab cluster-admin web panel\\n\\n[Service]\\nType=oneshot\\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\\n\\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\\n[Service]\\nExecStart=\\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-cluster-admin-webpanel-render.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel-render.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\\n[Unit]\\nDescription=Refresh Homelab cluster-admin web panel\\n\\n[Timer]\\nOnBootSec=1min\\nOnUnitActiveSec=1min\\nPersistent=true\\n\\n[Install]\\nWantedBy=timers.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-cluster-admin-webpanel.service\",\"Group\":\"\",\"LoadState\":\"loaded\",\"SubState\":\"running\",\"TriggeredBy\":\"\",\"Triggers\":\"\",\"UnitFileState\":\"enabled\",\"User\":\"\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-cluster-admin-webpanel.service\",\"unit_text\":\"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\\n[Unit]\\nDescription=Homelab cluster-admin web panel\\nAfter=network-online.target\\nWants=network-online.target\\n\\n[Service]\\nType=simple\\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\\nRestart=always\\nRestartSec=5\\n\\n[Install]\\nWantedBy=multi-user.target\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"failed\",\"DropInPaths\":\"\",\"ExecStart\":\"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.service\",\"Group\":\"root\",\"LoadState\":\"loaded\",\"SubState\":\"failed\",\"TriggeredBy\":\"homelab-stage4i-verify.timer\",\"Triggers\":\"\",\"UnitFileState\":\"static\",\"User\":\"root\",\"WorkingDirectory\":\"\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.service\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.service\\n[Unit]\\nDescription=Verify active immutable Stage 4I task version\\nAfter=local-fs.target\\n\\n[Service]\\nType=oneshot\\nUser=root\\nGroup=root\\nExecStart=/usr/local/sbin/homelab-stage4i-verify\\nNoNewPrivileges=true\\nPrivateDevices=true\\nProtectClock=true\\nProtectControlGroups=true\\nProtectHome=true\\nProtectKernelLogs=true\\nProtectKernelModules=true\\nProtectKernelTunables=true\\nRestrictNamespaces=true\\nRestrictRealtime=true\\nLockPersonality=true\\nMemoryDenyWriteExecute=true\\nUMask=0027\"},{\"cat_rc\":0,\"show\":{\"ActiveState\":\"active\",\"DropInPaths\":\"\",\"FragmentPath\":\"/etc/systemd/system/homelab-stage4i-verify.timer\",\"LoadState\":\"loaded\",\"SubState\":\"waiting\",\"TriggeredBy\":\"\",\"Triggers\":\"homelab-stage4i-verify.service\",\"UnitFileState\":\"enabled\"},\"show_rc\":0,\"unit\":\"homelab-stage4i-verify.timer\",\"unit_text\":\"# /etc/systemd/system/homelab-stage4i-verify.timer\\n[Unit]\\nDescription=Regularly verify active immutable Stage 4I task version\\n\\n[Timer]\\nOnBootSec=2min\\nOnUnitActiveSec=15min\\nAccuracySec=1min\\nPersistent=true\\nUnit=homelab-stage4i-verify.service\\n\\n[Install]\\nWantedBy=timers.target\"}],\"webroots\":[]},\"remote_target\":{\"evidence\":[{\"line\":21,\"target_count\":1,\"text\":\"scp -q -o BatchMode=yes -o ConnectTimeout=8 \\\"$TMP\\\"/*.txt debian@:/tmp/\"},{\"line\":24,\"target_count\":1,\"text\":\"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@ 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \\\"$f\\\" && sudo install -m 0644 \\\"$f\\\" /var/lib/homelab-health/pve01-pushed/; done'\"}],\"target_cardinality\":1,\"target_host_is_private_ipv4\":true,\"target_host_sha256\":\"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69\",\"target_user\":\"debian\"},\"rollback_restored\":false,\"rollback_started\":false,\"schema\":15,\"status\":\"OK\",\"transaction_gate\":{\"application_architecture_found\":true,\"next_required_action\":\"build one corrected local+remote atomic transaction\",\"pushed_health_reference_found\":true,\"ready_for_corrected_atomic_transaction\":true,\"remote_probe_root\":true},\"transport\":{\"attempts\":[{\"privilege\":\"sudo-root\",\"rc\":0,\"stderr\":\"\",\"stderr_bytes\":0,\"stderr_sha256\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"stdout_bytes\":82754,\"stdout_sha256\":\"603f5eb85c971d0bc029287cf91d9d0239b132dd409df5bbf8c5f47a7bb32738\"}],\"privilege\":\"sudo-root\",\"success\":true},\"v14_exact_result\":{\"changes_made\":false,\"control_plane_error\":null,\"identity_confirmed\":true,\"mutation_outcome\":\"NO_MUTATION\",\"output_bytes\":10768,\"output_sha256\":\"cc30d7f4e6be0ab5e78545d4537a39178b59e9f1a79227efaec134b78f6fc233\",\"rollback_restored\":null,\"rollback_started\":false,\"runner_document_rc\":0,\"runner_json_sha256\":\"c932bbad4dd7f176e29621f3c549e2571baf7f84500ef2c5408a04c6a84083ff\",\"runner_rc\":0,\"runner_status\":\"OK\",\"runner_text_sha256\":\"70796c2456365c9224dd6ff02897d29412349eb1a5fea03c89ae9212217a03f8\",\"runner_wrapper_matches_document\":true,\"state_document_sha256\":\"f3db8a869123d1266f906fdd02ca18a890141c2bd63e2bc57a3a0e7c5962f319\",\"state_path_sha256\":\"2c050c8423a48aab8f448c94123e67f132381be3232e1d10274f4e87e1013b16\",\"state_status\":\"OK\"}}\n"
+  "output": "HOMELAB_RESULT_CONTRACT={\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z\",\"command_rc\":0,\"mutation_outcome\":\"NO_MUTATION\",\"output_truncated_in_json\":false,\"rc\":0,\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\",\"version\":1}\n{\"bounded_adapter_scan\":null,\"changes_made\":false,\"command_id\":\"HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z\",\"command_rc\":0,\"evidence_bundle\":{\"bundle_bytes_ok\":true,\"bundle_sha256_ok\":true,\"exists\":true,\"member_count\":6,\"output_member_matches\":0,\"path_is_exact_from_previous_result\":true,\"runner_json_member_matches\":1,\"runner_text_member_matches\":1,\"unsafe_member_rejected\":false},\"files_written\":false,\"mutation_outcome\":\"NO_MUTATION\",\"network_requests_made\":false,\"next_action\":\"RUN_ONE_READ_ONLY_GITEA_DEFINED_CLUSTER_FACT_AND_DOCUMENT_GAP_ANALYSIS\",\"output_truncated_in_json\":false,\"pre_command_known_error_gate\":{\"checked\":true,\"count\":28,\"ids\":[\"candidate-file-limit-600-20260805\",\"runner-wrapper-mismatch-after-success-20260805\",\"canonical-cluster-handbook-ambiguous-20260805\",\"pvesh-adapter-context-unreadable-20260805\",\"homelab-ops-bare-repository-not-found-local-20260805\",\"gitea-network-fallback-stage-not-emitted-20260805\",\"gitea-owner-hardcoded-homelab-admin-20260805\",\"gitea-git-auth-required-or-credential-unavailable-20260805\",\"gitea-process-self-match-payload-name-20260805\",\"sanitize-bool-attributeerror-20260805\",\"reader-v3-path-misclassified-as-route-20260805\",\"reader-v3-loopback-assumption-20260805\",\"reader-v3-capability-not-treated-as-secret-20260805\",\"reader-v3-systemd-unit-fed-to-python-ast-20260805\",\"reader-v3-helper-control-flow-not-traced-20260805\",\"reader-v3-post-capability-operation-not-derived-20260805\",\"reader-v3-detected-operations-not-tested-20260805\",\"reader-v3-manifest-global-not-bound-in-sandbox-20260805\",\"reader-v3-result-file-second-source-not-read-20260805\",\"reader-v3-negative-control-mixed-with-production-20260805\",\"reader-v3-manifest-schema-guessed-20260805\",\"reader-result-output-bound-after-compaction-20260805\",\"reader-v3-generation-record-shape-guessed-20260805\",\"reader-v3-result-file-assumed-standalone-json-20260805\",\"reader-v3-gitea-detail-marker-false-inventory-20260805\",\"reverse-proxy-runtime-not-visible-on-host-20260805\",\"gitea-public-owner-not-anonymously-discoverable-20260805\",\"exact-output-long-line-not-projectable-by-portal-view-20260805\"]},\"projection\":{\"analysis_outcome\":{\"error_fingerprint\":\"26d65952ac8f64197a6fd76ec56e3b5ffda8c4838efe2af2947a603d01270b92\",\"next_action\":\"RUN_ONE_READ_ONLY_GITEA_DEFINED_CLUSTER_FACT_AND_DOCUMENT_GAP_ANALYSIS\",\"ready_for_gitea_defined_cluster_fact_analysis\":true,\"root_cause_classification\":\"GITEA_CLUSTER_HANDBOOK_AND_ERROR_SYSTEM_DEEPLY_ANALYZED\",\"status\":\"OK\"},\"canonical_handbook\":{\"candidates\":[{\"canonical_phrase_hits\":3,\"canonical_score\":227,\"content_sha256\":\"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64\",\"headings\":[],\"inbound_reference_count\":8,\"path\":\"errors/regression-cases.json\"},{\"canonical_phrase_hits\":0,\"canonical_score\":74,\"content_sha256\":\"a49bcb32ac7212d7fb9768db8a277cb5525c873c84dd6fb633edae35dc6de532\",\"headings\":[],\"inbound_reference_count\":0,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh\"},{\"canonical_phrase_hits\":1,\"canonical_score\":56,\"content_sha256\":\"ac5dc881dcd9fcb72644bc893e866f463c9c2b9b4544e32434e21b5d34168fb5\",\"headings\":[],\"inbound_reference_count\":1,\"path\":\"inventory/access-paths.json\"},{\"canonical_phrase_hits\":0,\"canonical_score\":55,\"content_sha256\":\"666072447ba11d346772f62f274dadc974c4463e20f67e67f82f228f1fcf2ce8\",\"headings\":[],\"inbound_reference_count\":0,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/rollback.sh\"},{\"canonical_phrase_hits\":0,\"canonical_score\":54,\"content_sha256\":\"636254d2526f0f62184d47cdb4c30c2831b11869c2000c557666f60b55d7691b\",\"headings\":[],\"inbound_reference_count\":0,\"path\":\"tasks/cluster-cloud-quorum-vm160-canary-v1/task.json\"}],\"path\":\"errors/regression-cases.json\",\"status\":\"UNIQUE_CONFIRMED\"},\"documentation_coverage\":{\"access\":{\"file_count\":22,\"paths\":[\"changes/CR-2026-0018/regression-cases-draft.json\",\"docs/planned/skladchik-moderator-assistant.md\",\"errors/regression-cases.json\",\"inventory/access-paths.json\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/dry-run.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/prepare.sh\"]},\"backup\":{\"file_count\":29,\"paths\":[\"docs/ARCHITECTURE.md\",\"docs/skladchik-cookie-refresh.md\",\"errors/regression-cases.json\",\"observed/cluster-guests.json\",\"policies/cluster-cloud-quorum.json\",\"schemas/change.schema.json\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/assets/pve02-vm160-preflight-probe.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/assets/vm160-cloud-quorum-worker.sh\"]},\"error_system\":{\"file_count\":44,\"paths\":[\".gitea/workflows/ci.yml\",\"changes/CR-2026-0018/caller-inventory-requirements.md\",\"changes/CR-2026-0018/regression-cases-draft.json\",\"changes/README.md\",\"docs/ARCHITECTURE.md\",\"docs/skladchik-cookie-refresh.md\",\"docs/WORKFLOW.md\",\"errors/regression-cases.json\"]},\"internal_external\":{\"file_count\":7,\"paths\":[\"changes/CR-2026-0018/regression-cases-draft.json\",\"errors/regression-cases.json\",\"policies/operating-policy.json\",\"tasks/homelab-logrotate-namespace-latch-clearance-v1/README.txt\",\"tests/test_cr_2026_0005.py\",\"tests/test_cr_2026_0011_skladchik_cookie_jar_reset.py\",\"tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py\"]},\"repair\":{\"file_count\":31,\"paths\":[\"changes/CR-2026-0018/caller-inventory-requirements.md\",\"changes/README.md\",\"docs/ARCHITECTURE.md\",\"docs/skladchik-cookie-refresh.md\",\"docs/WORKFLOW.md\",\"errors/regression-cases.json\",\"policies/cluster-cloud-quorum.json\",\"policies/operating-policy.json\"]},\"services\":{\"file_count\":21,\"paths\":[\"changes/CR-2026-0018/caller-inventory-requirements.md\",\"docs/ARCHITECTURE.md\",\"docs/planned/skladchik-moderator-assistant.md\",\"errors/regression-cases.json\",\"observed/cluster-guests.json\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/assets/vm160-cloud-quorum-worker.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/dry-run.sh\"]}},\"enumeration\":{\"candidate_file_count\":50,\"fetch_failure_count\":0,\"fetched_file_count\":50,\"fetched_total_bytes\":173536,\"tree_blob_count\":125,\"tree_complete\":true,\"tree_directory_count\":36,\"tree_entry_count\":161},\"error_system_current\":{\"duplicate_error_ids\":[],\"duplicate_fingerprints\":[],\"enforcement_path_count\":7,\"enforcement_paths\":[\"schemas/change.schema.json\",\"schemas/context.schema.json\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/assets/pve02-vm160-preflight-probe.sh\",\"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh\",\"tasks/homelab-logrotate-namespace-latch-clearance-v1/phases/preflight.sh\",\"tasks/skladchik-reports-monitor-controlled-reauth-v1/assets/edge-preflight.sh\",\"tasks/skladchik-reports-monitor-controlled-reauth-v1/phases/preflight.sh\"],\"gate_signals\":{\"ci_workflow_present\":true,\"duplicate_failed_command_block_signal\":false,\"error_register_checked_signal\":false,\"fingerprint_signal\":false,\"preflight_references_error_register\":true,\"reference_register_checked_signal\":false,\"schema_validation_signal\":true,\"tests_present\":true},\"machine_parseable_registry_file_count\":2,\"prevention_evidence_count\":0,\"registry_path_count\":2,\"registry_paths\":[\"changes/CR-2026-0018/regression-cases-draft.json\",\"errors/regression-cases.json\"],\"regression_path_count\":2,\"regression_paths\":[\"changes/CR-2026-0018/regression-cases-draft.json\",\"errors/regression-cases.json\"],\"superseded_path_count\":0,\"superseded_paths\":[],\"test_path_count\":14,\"test_paths\":[\"tests/fixtures/gitea-legacy-auth-permission.json\",\"tests/test_cr_2026_0005.py\",\"tests/test_cr_2026_0006.py\",\"tests/test_cr_2026_0006_vm160_task.py\",\"tests/test_cr_2026_0007_skladchik_reauth.py\",\"tests/test_cr_2026_0007_skladchik_semantic_repair.py\",\"tests/test_cr_2026_0008_skladchik_snapshot_integrity.py\",\"tests/test_cr_2026_0009_skladchik_concurrency_guard.py\",\"tests/test_cr_2026_0011_skladchik_cookie_jar_reset.py\",\"tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py\"],\"unique_error_id_count\":61,\"unique_fingerprint_count\":0,\"workflow_path_count\":1,\"workflow_paths\":[\".gitea/workflows/ci.yml\"]},\"gaps\":[\"no_static_evidence_error_fingerprint_gate\"],\"pre_command_gate\":{\"checked\":true,\"projection_command_known_error_count\":28,\"regression_tests_passed\":true,\"reported_error_count\":27},\"previous_exact_result\":{\"capsule_sha256\":\"031aa6c1029c24d8b7dac8192707376fa67ee93cdcfceedcfb768a0b7d4ceb8e\",\"command_id\":\"HOMELAB-CLUSTER-KB-V1-GITEA-REPOSITORY-CONTENT-DEEP-ANALYSIS-READ-ONLY-20260805T124500Z\",\"identity_fully_verified\":true,\"output_bytes\":27219,\"output_sha256\":\"ead28fd156e646c27f6e988c12ea1e89a498882f07071faacba670e87c8a7983\",\"runner_json_sha256\":\"e3211267d27f7497339cd34ecdbe0cce21d039965625ffd5f3236dc20513b082\",\"runner_text_sha256\":\"083286377044254ff016f0f9024655e228a67ae2737d7e1d9641996d5af8e09a\"},\"previous_result_contract\":{\"changes_made\":false,\"command_rc\":0,\"mutation_outcome\":\"NO_MUTATION\",\"output_truncated_in_json\":false,\"rc\":0,\"rollback_restored\":null,\"rollback_started\":false,\"status\":\"OK\"},\"recommended_error_system_v2\":{\"acceptance_criteria\":[\"zero hard-coded owner/path/host/version unless proven by exact prior evidence or current Gitea content\",\"zero mutation without byte-for-byte backup and tested rollback\",\"zero repeated known error fingerprint\",\"all command outputs bounded and non-truncated\",\"all new errors recorded before any subsequent command\"],\"components\":[{\"component\":\"machine_error_registry\",\"requirement\":\"one append-only machine-readable record per proven failure with stable id, normalized fingerprint, exact command/capsule/result SHA, root cause, correction and prevention test\"},{\"component\":\"pre_command_gate\",\"requirement\":\"before every .hcmd, read canonical cluster reference and entire active error registry; block duplicate failed fingerprints and commands that violate a prevention test\"},{\"component\":\"transaction_manifest\",\"requirement\":\"every mutation declares exact source SHA, candidate SHA, byte backup SHA, preconditions, atomic apply, verification and deterministic rollback\"},{\"component\":\"post_result_ingestion\",\"requirement\":\"after exact-result verification, automatically classify the outcome, append new proven failures, close corrected cases and link superseded commands\"},{\"component\":\"regression_suite\",\"requirement\":\"positive and negative fixtures for every recorded error; payload self-test and repository CI must pass before command publication\"},{\"component\":\"knowledge_freshness\",\"requirement\":\"canonical handbook stores last verified fact source, verification time, source SHA and stale/unknown markers; commands may not rely on stale or unknown values\"}],\"grounded_in_detected_gaps\":[\"no_static_evidence_error_fingerprint_gate\"],\"minimum_gate_order\":[\"verify previous exact result identity\",\"read canonical handbook at exact branch commit\",\"read active errors and prevention tests\",\"validate target facts from authoritative source\",\"run payload regression tests\",\"only then publish or execute the next command\"],\"name\":\"Error System V2: evidence-first command gate\",\"reuse_existing_error_root\":\"changes/CR-2026-0018\"},\"repository\":{\"branch_commit_sha\":\"fe1221b06643db3b00a621b0230f92a46a4edc2b\",\"default_branch\":\"main\",\"full_name\":\"homelab-admin/homelab-ops\",\"private\":true},\"source_result_projection\":{\"budget_bytes\":29232,\"bytes\":26771,\"max_bytes\":30000,\"steps\":[\"top_analyzed_files_12\",\"handbook_candidates_6\"]}},\"projection_bytes_before_final_wrapper\":11730,\"rc\":0,\"read_only\":true,\"regression_tests\":{\"duplicate_sha_string_rejected\":true,\"passed\":true,\"recursive_command_identity\":true,\"unique_sha_string_extraction\":true},\"rollback_restored\":null,\"rollback_started\":false,\"root_cause_classification\":\"EXACT_PRIOR_GITEA_DEEP_ANALYSIS_OUTPUT_PROJECTED\",\"schema\":1,\"status\":\"OK\"}\n"
 }
diff --git a/runtime/latest.txt b/runtime/latest.txt
index 04c4bc8..e3f74ea 100644
--- a/runtime/latest.txt
+++ b/runtime/latest.txt
@@ -1,15 +1,15 @@
 CHAT_OUTPUT_BEGIN
-COMMAND_ID=HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z
+COMMAND_ID=HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z
 STATUS=OK
 RC=0
 HOST=pve01
 MODE=verify
-COMPONENT=homelab-cluster-admin-update-visibility
+COMPONENT=cluster-knowledge-base-error-system
 REFERENCE_REGISTER_CHECK=OK
 REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
 ERROR_REGISTER_CHECK=OK
 ERROR_REGISTER_SHA256=24934a2c7fa5c26f6e828583c4aefac7f143b40c0171736ed7571e617d55ab83
-COMMAND_SHA256=4845b4e99e60efaa533b2a551d4bf1773085ce96ef0e4ce9737b920907b0553f
+COMMAND_SHA256=8d626b3d942e0c8ad8135e1fbf4148a82c5df736a52cc969eb0f6ae022f1ee59
 DUPLICATE_FAILED_COMMAND_BLOCKED=false
 EXECUTION_STARTED=true
 CHANGE_DECLARED=false
@@ -24,11 +24,11 @@ MUTATION_OUTCOME=NO_MUTATION
 SANITIZED=yes
 SECRETS_INCLUDED=no
 PRIVATE_ADDRESSES_INCLUDED=no
-RAW_EVIDENCE_SHA256=b672b2fde18de662e73d8a98d4cc036895f9de2007679edc3c085392050e66e6
-SANITIZED_OUTPUT_SHA256=c6579c5f1af9da00000354e8fb5aa3ae7128dcfad766bb4e07c56cef356fd2e8
+RAW_EVIDENCE_SHA256=2a7ec04b2e7036e6cc5751216e516b7d78ef56dc135326379580ef58dbc66a29
+SANITIZED_OUTPUT_SHA256=2a7ec04b2e7036e6cc5751216e516b7d78ef56dc135326379580ef58dbc66a29
 OUTPUT_BEGIN
-HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z","command_rc":0,"control_plane_error":null,"mutation_outcome":"NO_MUTATION","rc":0,"ready_for_corrected_atomic_transaction":true,"remote_application_candidate_count":40,"remote_hostname":"cluster-admin","remote_index_count":1,"remote_probe_privilege":"sudo-root","remote_reference_file_count":1,"remote_target_cardinality":1,"remote_target_host_sha256":"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69","remote_unit_count":16,"remote_webroot_config_count":0,"report_bytes":85056,"report_sha256":"f00b483f5fc0f9520669edc3135ec323a055d71625aafdc1963aef3a95356fbf","rollback_restored":false,"rollback_started":false,"status":"OK","v14_exact_result_identity_confirmed":true,"version":1,"webpanel_script_sha256":"2c57cca503cb0734e5114c2ab46b892a810bdc40c2baa7387b4094373db19276"}
-{"changes_made":false,"command_id":"HOMELAB-CLUSTER-ADMIN-UPDATES-V15-REMOTE-ARCHITECTURE-READ-ONLY-20260805T130000Z","mutation_outcome":"NO_MUTATION","read_only":true,"remote_architecture":{"application_candidates":[{"bytes":133,"matches":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},{"bytes":149,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},{"bytes":154,"matches":[{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},{"bytes":151,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},{"bytes":231,"matches":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},{"bytes":154,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},{"bytes":138,"matches":[{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},{"bytes":127,"matches":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},{"bytes":616,"matches":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},{"bytes":1190,"matches":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},{"bytes":223,"matches":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},{"bytes":803,"matches":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},{"bytes":131,"matches":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},{"bytes":301,"matches":[{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},{"bytes":89,"matches":[{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f"},{"bytes":8444,"matches":[{"line":4,"text":"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')"},{"line":64,"text":" return f\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\n\""},{"line":82,"text":" old="},{"line":97,"text":"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)"},{"line":99,"text":"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))"}],"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a"},{"bytes":28599,"matches":[{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE ="},{"line":14,"text":"COOKIE ="},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":20,"text":"    return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":73,"text":"    return f\"\"\""},{"line":152,"text":"    return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":220,"text":"    page = f\"\"\"Cluster Admin Incident Engine
Cluster Admin Incident Engine {engine_badge}
observe/notify · auto-heal отключён
{cards}
{nodes_html}{incidents_html}{services_html}
\"\"\""},{"line":243,"text":"        \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":"        \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":"        \"cluster_admin_up 1\","},{"line":246,"text":"        \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":"        \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":"        \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":"        \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":"        \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":"        \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":"        \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":"        \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":"        \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":"        \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":"        \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":"        \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":"        \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":"        \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":"        \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":"        \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":"        \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":"        \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":267,"text":"        lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6"},{"bytes":922,"matches":[{"line":2,"text":"first = getpass.getpass(\"Новый пароль Cluster Admin: \")"},{"line":12,"text":"directory = \"/etc/cluster-admin-incident-engine\""},{"line":16,"text":"account = pwd.getpwnam(\"clusteradmin\")"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3"},{"bytes":10640,"matches":[{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":36,"text":"    data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":"    connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":41,"text":"        cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":50,"text":"            if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":57,"text":"        cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":108,"text":"    line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe"},{"bytes":1857,"matches":[{"line":3,"text":"  \"type\": \"cluster-admin-stage1-foundation-migration-plan\","},{"line":43,"text":"    \"function\": \"cluster_admin_enrich_collector_run\","}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670"},{"bytes":1342,"matches":[{"line":3,"text":"  \"collector_call\": \"SELECT cluster_admin_sync_foundation(%s)\","},{"line":15,"text":"  \"function\": \"cluster_admin_sync_foundation\","},{"line":16,"text":"  \"function_signature\": \"cluster_admin_sync_foundation(bigint)\","},{"line":26,"text":"  \"type\": \"cluster-admin-stage2-migration-plan\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c"},{"bytes":2355,"matches":[{"line":51,"text":"  \"type\": \"cluster-admin-stage2-dynamic-acceptance-contract\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588"},{"bytes":2832,"matches":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"Homelab Cluster Admin"},{"line":58,"text":"
Homelab Cluster Admin
"},{"line":74,"text":"
Service map tail
$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)
"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},{"bytes":3471,"matches":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},{"bytes":1433,"matches":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},{"bytes":848,"matches":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":10353,"matches":[{"line":7,"text":"Homelab Cluster Admin"},{"line":19,"text":"
Homelab Cluster Admin
"},{"line":24,"text":"
Self
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"
Observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
"},{"line":28,"text":"
Full observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
"},{"line":29,"text":"
Mail Cloud backup
STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md
"},{"line":32,"text":"
"},{"line":33,"text":"
Final readiness
STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md
"},{"line":34,"text":"
Strict seal
STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md
"},{"line":35,"text":"
Service map tail
# Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":46,"text":"OK severity="},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a"},{"bytes":168,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify"}],"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729"},{"bytes":259,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741"},{"bytes":134,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517"},{"bytes":141,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6"},{"bytes":189,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941"},{"bytes":131,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0"}],"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":199,"matches":[{"line":1,"text":"STATUS="}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680"},{"bytes":272,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec"},{"bytes":380,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"}],"euid":0,"gate":{"application_candidate_count":40,"ready_for_remote_transaction_design":true,"reference_file_count":1,"unit_count":16},"health_input_files":[{"bytes":380,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":217,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"},{"bytes":128,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac"},{"bytes":1535,"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b"}],"hostname":"cluster-admin","indexes":[{"bytes":10353,"path":"/var/www/homelab-cluster-admin/index.html","root":"/var/www/homelab-cluster-admin","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a","title_lines":["Homelab Cluster Admin","
Homelab Cluster Admin
","
Self
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active","
Observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
","
Full observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
","
Mail Cloud backup
STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md
","
","
Final readiness
STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md
","
Strict seal
STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md
","
Service map tail
# Homelab cluster-admin full observer","pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2","pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"]}],"listeners":{"rc":0,"stderr":"","stdout":"State  Recv-Q Send-Q Local Address:Port Peer Address:PortProcess                                   \nLISTEN 0      5            0.0.0.0:8080      0.0.0.0:*    users:((\"python3\",pid=439,fd=3))         \nLISTEN 0      2048         0.0.0.0:8081      0.0.0.0:*    users:((\"python3\",pid=487,fd=6))         \nLISTEN 0      244        127.0.0.1:5432      0.0.0.0:*    users:((\"postgres\",pid=469,fd=6))        \nLISTEN 0      4096   127.0.0.53%lo:53        0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=18))\nLISTEN 0      4096         0.0.0.0:5355      0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=12))\nLISTEN 0      4096      127.0.0.54:53        0.0.0.0:*    users:((\"systemd-resolve\",pid=380,fd=20))\nLISTEN 0      128          0.0.0.0:22        0.0.0.0:*    users:((\"sshd\",pid=457,fd=3))            \nLISTEN 0      244            [::1]:5432         [::]:*    users:((\"postgres\",pid=469,fd=5))        \nLISTEN 0      4096            [::]:5355         [::]:*    users:((\"systemd-resolve\",pid=380,fd=14))\nLISTEN 0      128             [::]:22           [::]:*    users:((\"sshd\",pid=457,fd=4))            "},"matching_files":[{"bytes":133,"matches":[{"line":2,"text":"Description=Homelab cluster-admin full observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/full-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","sha256":"8fc1e66ba031dc95caa304f4e1b1b3549be0c64a136e381f8a9cf2f926690c9b"},{"bytes":149,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.timer","sha256":"5a1040974ace21d885e230facf14fbeac87daedbb33a1fb2a6b0e77b1bc4657b"},{"bytes":154,"matches":[{"line":2,"text":"Description=Refresh Homelab cluster-admin web panel"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","sha256":"c3c322cf9b17495f735f6648665da767fce8125d9d13e2e4bf7a9e13c6e76311"},{"bytes":151,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin self check"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","sha256":"549eacb0871d0c04cf5ec420c3f23776f30b89cdda7635dc78fc40a2fb5c9ea1"},{"bytes":231,"matches":[{"line":2,"text":"Description=Run Cluster Admin incident collector every minute"},{"line":9,"text":"Unit=cluster-admin-incident-engine-collector.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","sha256":"aa43fc0471111d99cd2b61a34d757beb0ce58f73ae9634ddb00b1a7b41165bae"},{"bytes":154,"matches":[{"line":2,"text":"Description=Run homelab cluster-admin full observer"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","sha256":"79bab9bc35daa4b38939747425d1e3fc259f32a132f6a66e191214ab92723d23"},{"bytes":138,"matches":[{"line":2,"text":"Description=Render Homelab cluster-admin web panel"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","sha256":"86f57a6ca0d1a434d9a0f272f08d9587e7a39c68196536bbaeb58dce46a5877e"},{"bytes":127,"matches":[{"line":2,"text":"Description=Homelab cluster-admin self check"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/self-check.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-self-check.service","sha256":"50d3e1267ea5f389f3c45ca20764ac74fb6038554cb87072ba2d0498a8c88a3d"},{"bytes":616,"matches":[{"line":2,"text":"Description=Cluster Admin inventory metrics and incident collector"},{"line":3,"text":"After=network-online.target postgresql.service cluster-admin-incident-engine.service"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py"},{"line":18,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":19,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","sha256":"1b879987e540fe1db10e4fd37c036f252c1f0e07b4122b9a6a32c955f51775f7"},{"bytes":1190,"matches":[{"line":2,"text":"Description=Cluster Admin restricted read-only probe controller"},{"line":8,"text":"User=clusteradmin"},{"line":9,"text":"Group=clusteradmin"},{"line":10,"text":"Environment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt"},{"line":11,"text":"ExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run"},{"line":37,"text":"ReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent"},{"line":38,"text":"ReadWritePaths=/var/lib/cluster-admin-probe-agent"},{"line":39,"text":"StateDirectory=cluster-admin-probe-agent"},{"line":41,"text":"RuntimeDirectory=cluster-admin-probe-agent"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.service","sha256":"599cc481fc1b6097c0cdb88fa6c0e501beca5940021e2745c8188ab9ec835771"},{"bytes":223,"matches":[{"line":2,"text":"Description=Run Cluster Admin restricted probes every minute"},{"line":9,"text":"Unit=cluster-admin-probe-agent.service"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-probe-agent.timer","sha256":"4117c55ca2db2cc5e41a173a2a1e55eca1c9867a00b8e64b53c3598966d3976d"},{"bytes":803,"matches":[{"line":2,"text":"Description=Cluster Admin Incident Engine web application"},{"line":9,"text":"User=clusteradmin"},{"line":10,"text":"Group=clusteradmin"},{"line":11,"text":"WorkingDirectory=/opt/cluster-admin-incident-engine"},{"line":12,"text":"EnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf"},{"line":21,"text":"ReadOnlyPaths=/etc/cluster-admin-incident-engine"},{"line":22,"text":"ReadWritePaths=/var/lib/cluster-admin-incident-engine"}],"mode":"0o644","path":"/etc/systemd/system/cluster-admin-incident-engine.service","sha256":"1892e494de5caab11100d5ea9257b82857fd6138d73cfae91e888759321745ea"},{"bytes":131,"matches":[{"line":2,"text":"Description=Homelab cluster-admin observer"},{"line":6,"text":"ExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-observer.service","sha256":"8bc7e07804ec68a402b58b6442943f6f4a41931826bcbd4631d7818484500042"},{"bytes":301,"matches":[{"line":2,"text":"Description=Homelab cluster-admin web panel"},{"line":8,"text":"ExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","sha256":"84b6f85e83c0a556cc1a7ce7686ad25932e6c8a089a99b1f97cd8203b4247e5b"},{"bytes":89,"matches":[{"line":3,"text":"ExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"}],"mode":"0o644","path":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","sha256":"54a4f7ac436387a9952273e8d34bf9de4ed29c8d60e76c0789bcb6945936551f"},{"bytes":8444,"matches":[{"line":4,"text":"POL=Path('/etc/cluster-admin-probe-agent/policy.json'); STATE=Path('/var/lib/cluster-admin-probe-agent'); HEALTH=Path('/var/lib/homelab-health/cluster-admin-probe-agent.txt')"},{"line":64,"text":" return f\"STATUS={'OK' if bad==0 else 'BAD'} TS={now()} TYPE=cluster-admin-probe-agent BAD={bad} WARN=0 MODE=observe-notify TARGETS={len(rs)} PROBE_OK={ok} PROBE_BAD={bad}\\n\""},{"line":82,"text":" old="},{"line":97,"text":"  p=read(os.environ.get('CLUSTER_ADMIN_PROBE_POLICY',str(POL))); check_policy(p)"},{"line":99,"text":"  STATE=Path(os.environ.get('CLUSTER_ADMIN_PROBE_STATE',str(STATE))); HEALTH=Path(os.environ.get('CLUSTER_ADMIN_PROBE_HEALTH',str(HEALTH)))"}],"mode":"0o750","path":"/opt/cluster-admin-probe-agent/controller.py","sha256":"5d34a863ff5f17161e5ed52f1e208b093e015775d0d693e0f373edc5b3c5ac1a"},{"bytes":28599,"matches":[{"line":12,"text":"KEY = open(\"/etc/cluster-admin-incident-engine/web.key\", \"rb\").read().strip()"},{"line":13,"text":"PASSFILE ="},{"line":14,"text":"COOKIE ="},{"line":17,"text":"app.add_middleware(TrustedHostMiddleware, allowed_hosts=[\"pvepro.ru\",\"www.pvepro.ru\",\"cluster-admin.vpn.gram1.ru\",\"[PRIVATE_IP]\",\"127.0.0.1\",\"localhost\"])"},{"line":20,"text":"    return psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":73,"text":"    return f\"\"\""},{"line":152,"text":"    return {\"status\":\"OK\",\"service\":\"cluster-admin-incident-engine\",\"mode\":\"observe-notify\"}"},{"line":220,"text":"    page = f\"\"\"Cluster Admin Incident Engine
Cluster Admin Incident Engine {engine_badge}
observe/notify · auto-heal отключён
{cards}
{nodes_html}{incidents_html}{services_html}
\"\"\""},{"line":243,"text":"        \"# HELP cluster_admin_up Cluster Admin Incident Engine availability\","},{"line":244,"text":"        \"# TYPE cluster_admin_up gauge\","},{"line":245,"text":"        \"cluster_admin_up 1\","},{"line":246,"text":"        \"# HELP cluster_admin_nodes_total Discovered Proxmox nodes\","},{"line":247,"text":"        \"# TYPE cluster_admin_nodes_total gauge\","},{"line":248,"text":"        \"cluster_admin_nodes_total \" + str(nodes),"},{"line":249,"text":"        \"# HELP cluster_admin_guests_total Discovered virtual machines and containers\","},{"line":250,"text":"        \"# TYPE cluster_admin_guests_total gauge\","},{"line":251,"text":"        \"cluster_admin_guests_total \" + str(guests),"},{"line":252,"text":"        \"# HELP cluster_admin_services_total Discovered services\","},{"line":253,"text":"        \"# TYPE cluster_admin_services_total gauge\","},{"line":254,"text":"        \"cluster_admin_services_total \" + str(services),"},{"line":255,"text":"        \"# HELP cluster_admin_observations_total Stored metric observations\","},{"line":256,"text":"        \"# TYPE cluster_admin_observations_total gauge\","},{"line":257,"text":"        \"cluster_admin_observations_total \" + str(observations),"},{"line":258,"text":"        \"# HELP cluster_admin_incidents_active_total Active correlated incidents\","},{"line":259,"text":"        \"# TYPE cluster_admin_incidents_active_total gauge\","},{"line":260,"text":"        \"cluster_admin_incidents_active_total \" + str(incidents),"},{"line":261,"text":"        \"# HELP cluster_admin_collector_age_seconds Seconds since last successful collector run\","},{"line":262,"text":"        \"# TYPE cluster_admin_collector_age_seconds gauge\","},{"line":263,"text":"        \"cluster_admin_collector_age_seconds \" + str(collector_age)"},{"line":267,"text":"        lines.append(\"cluster_admin_incidents_active{severity=\" + chr(34) + clean + chr(34) + \"} \" + str(count))"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/app.py","sha256":"17d95ebfc28cef34846a8999664019d5d032ee8176b7fedd9ceeebfc5c97e3b6"},{"bytes":922,"matches":[{"line":2,"text":"first = getpass.getpass(\"Новый пароль Cluster Admin: \")"},{"line":12,"text":"directory = \"/etc/cluster-admin-incident-engine\""},{"line":16,"text":"account = pwd.getpwnam(\"clusteradmin\")"}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/set-password.py","sha256":"083aacefce814d03175d1edc89fad8bb6fb0d3522400cf2746eb225941dfdfe3"},{"bytes":10640,"matches":[{"line":9,"text":"INVENTORY = \"/var/lib/cluster-admin-incident-engine/incoming/inventory.json\""},{"line":10,"text":"HEALTH = \"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt\""},{"line":12,"text":"HOSTS = {\"[PRIVATE_IP]\":\"pve01\",\"[PRIVATE_IP]\":\"pve02\",\"[PRIVATE_IP]\":\"pve03\",\"[PRIVATE_IP]\":\"edge-vm\",\"[PRIVATE_IP]\":\"nextcloud\",\"[PRIVATE_IP]\":\"forum-prod\",\"[PRIVATE_IP]\":\"core-apps\",\"[PRIVATE_IP]\":\"monitoring\",\"[PRIVATE_IP]\":\"cluster-admin\"}"},{"line":36,"text":"    data = json.load(open(INVENTORY, encoding=\"utf-8\"))"},{"line":37,"text":"    connection = psycopg2.connect(dbname=\"clusteradmin\", user=\"clusteradmin\", host=\"/var/run/postgresql\")"},{"line":41,"text":"        cursor.execute(\"INSERT INTO collector_runs(collector,status,started_at,metadata) VALUES(%s,%s,now(),%s) RETURNING id\",(\"cluster-inventory-prometheus\",\"RUNNING\",Json({\"inventory_generated_at\":data.get(\"generated_at\")})))"},{"line":50,"text":"            if str(item.get(\"name\", \"\")) != \"cluster-admin-incident-engine\" and str(item.get(\"status\", \"\")).upper() != \"OK\":"},{"line":57,"text":"        cursor.execute(\"SELECT cluster_admin_sync_foundation(%s)\",(run_id,))"},{"line":108,"text":"    line = \"STATUS=OK TS=\" + datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + \" TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=\" + str(len(data.get(\"nodes\",[]))) + \" GUESTS=\" + str(len(data.get(\"guests\",[]))) + \" SERVICES=\" + str(len(data.get(\"services\",[]))) + \" METRICS=\" + str(metric_count) + \" OPEN_INCIDENTS=\" + str(open_count) + \" MODE=observe-notify\\n\""}],"mode":"0o750","path":"/opt/cluster-admin-incident-engine/collector.py","sha256":"5dda66a1878046223af9c28a393129b44991a8083b430a76302767b460e7cbbe"},{"bytes":1857,"matches":[{"line":3,"text":"  \"type\": \"cluster-admin-stage1-foundation-migration-plan\","},{"line":43,"text":"    \"function\": \"cluster_admin_enrich_collector_run\","}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/001_entity_source_policy_foundation.plan.json","sha256":"0dd4e4541ccb37efce8fde49d9f2b7258d04957ee4360abdde7572731fa9d670"},{"bytes":1342,"matches":[{"line":3,"text":"  \"collector_call\": \"SELECT cluster_admin_sync_foundation(%s)\","},{"line":15,"text":"  \"function\": \"cluster_admin_sync_foundation\","},{"line":16,"text":"  \"function_signature\": \"cluster_admin_sync_foundation(bigint)\","},{"line":26,"text":"  \"type\": \"cluster-admin-stage2-migration-plan\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.plan.json","sha256":"d01f905e4940be5d09e610aecc9afeb7c9170e86f12e50d1c904d16ec74ae64c"},{"bytes":2355,"matches":[{"line":51,"text":"  \"type\": \"cluster-admin-stage2-dynamic-acceptance-contract\""}],"mode":"0o644","path":"/opt/cluster-admin-incident-engine/migrations/002_canonical_foundation_sync.dynamic-contract.json","sha256":"7e62285fe43724f8c539b28dc871efd4553cd9f59cfad5876096bb80aed0c588"},{"bytes":2832,"matches":[{"line":3,"text":"LOCK=/run/homelab-cluster-admin-observer.lock"},{"line":5,"text":"flock -n 9 || { cat /var/lib/homelab-health/cluster-admin-observer.txt 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":8,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":9,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":11,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-observer.txt\""},{"line":35,"text":"  line=$(curl -fsS --max-time 5 \"$url\" 2>/tmp/cluster-admin-curl.err | head -n1)"},{"line":45,"text":"SELF_LINE=$(cat /var/lib/homelab-health/cluster-admin-self-check.txt 2>/dev/null)"},{"line":48,"text":"  echo \"# Homelab cluster-admin observer\""},{"line":85,"text":"check_http_status_ok pve01_overall http://[PRIVATE_IP]:9101/overall.txt"},{"line":88,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-observer BAD=$BAD WARN=$WARN HOST=$(hostname 2>/dev/null) SELF_FAILED_UNITS=$FAILED QGA=$QGA PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/cluster-observer.sh","sha256":"8ad616cb43a97aa92717619c02d252a383b9b2d8d8f0a52e625b3f982f2bd3d5"},{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"Homelab Cluster Admin"},{"line":58,"text":"
Homelab Cluster Admin
"},{"line":74,"text":"
Service map tail
$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)
"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"},{"bytes":3471,"matches":[{"line":4,"text":"ROOT=/var/lib/homelab-cluster-admin"},{"line":5,"text":"HEALTH_DIR=/var/lib/homelab-health"},{"line":6,"text":"MAP=/etc/homelab-cluster-admin/service-map.tsv"},{"line":8,"text":"HEALTH=\"$HEALTH_DIR/cluster-admin-full-observer.txt\""},{"line":9,"text":"LOCK=/run/homelab-cluster-admin-full-observer.lock"},{"line":12,"text":"flock -n 9 || { cat \"$HEALTH\" 2>/dev/null || echo \"STATUS=WARN TYPE=cluster-admin-full-observer BAD=1 REASON=lock_busy\"; exit 0; }"},{"line":14,"text":"K=/opt/homelab-cluster-admin/ssh/cluster-admin-probe_ed25519"},{"line":26,"text":"  echo \"# Homelab cluster-admin full observer\""},{"line":28,"text":"  echo \"policy=critical_failures_are_bad_optional_tcp_is_inventory_info\""},{"line":105,"text":"LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-full-observer BAD=$BAD WARN=0 INFO_OFF=$INFO_OFF PROBE_OK=$PROBE_OK PROBE_BAD=$PROBE_BAD MAP_TOTAL=$TOTAL CRITICAL_FAILS=$CRIT_FAIL OPTIONAL_INFO_OFF=$OPT_OFF SELF_FAILED_UNITS=$SELF_FAILED QGA=$QGA REPORT=$REPORT\""}],"mode":"0o755","path":"/opt/homelab-cluster-admin/full-observer.sh","sha256":"1e02b024989a6b3d05f3785c47144f2002522e8d173fe24ac0599d0f5536fb5e"},{"bytes":1433,"matches":[{"line":3,"text":"ORIGINAL=/opt/homelab-cluster-admin/render-webpanel.sh"},{"line":4,"text":"HEALTH=/var/lib/homelab-health/cluster-admin-webpanel.txt"},{"line":5,"text":"STATUS_FILE=/var/www/homelab-cluster-admin/status.txt"}],"mode":"0o750","path":"/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh","sha256":"8afb6e9e1610b4389928fc4e2e60b01c42112128bca8cde1e6fb73ff94e99792"},{"bytes":848,"matches":[{"line":16,"text":"echo \"STATUS=$STATUS TS=$TS TYPE=cluster-admin-self-check BAD=$BAD HOST=$HOST IP_OK=$IP_OK DNS_OK=$DNS_OK FAILED_UNITS=$FAILED QGA=$QGA\" | sudo tee /var/lib/homelab-health/cluster-admin-self-check.txt >/dev/null"},{"line":17,"text":"cat /var/lib/homelab-health/cluster-admin-self-check.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/self-check.sh","sha256":"3cfd53a97b363d880a5b3807ad5d1acee8b7f5589f99737778a47699a3b7b972"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/status.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":10353,"matches":[{"line":7,"text":"Homelab Cluster Admin"},{"line":19,"text":"
Homelab Cluster Admin
"},{"line":24,"text":"
Self
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"},{"line":27,"text":"
Observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md
"},{"line":28,"text":"
Full observer
STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
"},{"line":29,"text":"
Mail Cloud backup
STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md
"},{"line":32,"text":"
"},{"line":33,"text":"
Final readiness
STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md
"},{"line":34,"text":"
Strict seal
STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md
"},{"line":35,"text":"
Service map tail
# Homelab cluster-admin full observer"},{"line":37,"text":"policy=critical_failures_are_bad_optional_tcp_is_inventory_info"},{"line":40,"text":"pve01 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:27Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve01 FAILED_UNITS=3 ROOT_PCT=16 STAGING_PCT=53 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=3 CT_RUNNING=2"},{"line":41,"text":"pve02 ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:28Z TYPE=cluster-admin-node-probe BAD=0 HOST=pve02 FAILED_UNITS=0 ROOT_PCT=30 STAGING_PCT=75 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=2"},{"line":42,"text":"pve03 ssh_rc=0 STATUS=WARN TS=2026-08-05T12:53:31Z TYPE=cluster-admin-node-probe BAD=1 HOST=pve03 FAILED_UNITS=2 ROOT_PCT=59 STAGING_PCT=54 PVE_CLUSTER=active PVE_DAEMON=active PVE_PROXY=active PVE_STATD=active COROSYNC=active QUORATE=Yes QM_RUNNING=2 CT_RUNNING=0"},{"line":43,"text":"edge ssh_rc=0 STATUS=OK TS=2026-08-05T12:53:33Z TYPE=cluster-admin-edge-probe BAD=0 HOST=edge-vm FAILED_UNITS=0 ROOT_PCT=42 SSHD=active"},{"line":46,"text":"OK severity="},{"line":57,"text":"OK severity=critical kind=tcp name=cluster_admin_ssh target=[PRIVATE_IP] value=22 tcp_22_rc=0"},{"line":78,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md
"}],"mode":"0o644","path":"/var/www/homelab-cluster-admin/index.html","sha256":"1cfa0e1207ddd3844bcdb837cec525408cb30f30c3ce9ef1a9cf34826654832a"},{"bytes":168,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:55:28+00:00 TYPE=cluster-admin-incident-engine-collector BAD=0 NODES=3 GUESTS=12 SERVICES=49 METRICS=47 OPEN_INCIDENTS=0 MODE=observe-notify"}],"mode":"0o640","path":"/var/lib/homelab-health/cluster-admin-incident-engine-collector.txt","sha256":"02f7f39b4d57e223aad37615f44281c030618b3cca5cbccbc8076e6c4a3f3729"},{"bytes":259,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-full-observer BAD=3 WARN=0 INFO_OFF=7 PROBE_OK=2 PROBE_BAD=2 MAP_TOTAL=30 CRITICAL_FAILS=0 OPTIONAL_INFO_OFF=7 SELF_FAILED_UNITS=1 QGA=active REPORT=/var/lib/homelab-cluster-admin/full-observer-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-full-observer.txt","sha256":"85a9bcda2d3a89aff26a72690402de32b88eba94ab5d52d34f0b0be2ca1b2741"},{"bytes":134,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-self-check BAD=1 HOST=cluster-admin IP_OK=1 DNS_OK=1 FAILED_UNITS=1 QGA=active"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-self-check.txt","sha256":"5b64de12be02911434db04c28881368c2178cf34c56c99d2238b2ac6f28fd517"},{"bytes":141,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T22:08:08Z TYPE=cluster-admin-incident-engine-metrics BAD=0 HTTP=200 NODES=3 SERVICES=48 MODE=nonsensitive-prometheus"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-metrics.txt","sha256":"82644bcd3a3c5b8fe251917a1f68a6d779742b1cbe7b5e8ee50b73eea275ebc6"},{"bytes":189,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-10T20:47:13Z TYPE=cluster-admin-incident-engine-foundation BAD=0 MODE=observe-notify DATABASE=postgresql TABLES=11 AUTH_CONFIGURED=no OLD_PANEL_HTTP=200 PORT_8081=FREE"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-foundation.txt","sha256":"55c3d6bc4aa7c0557de4ae1f61034d9e774b9444c87c6eefe3fb97c22b885941"},{"bytes":131,"matches":[{"line":1,"text":"STATUS=OK TS=2026-08-05T12:54:58Z TYPE=cluster-admin-probe-agent BAD=0 WARN=0 MODE=observe-notify TARGETS=4 PROBE_OK=4 PROBE_BAD=0"}],"mode":"0o640","path":"/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt","sha256":"d832877eca96cc6115f6a25ccf5177645d85aeb4aa12f3668bf5765953fa8af6"},{"bytes":214,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:55:27Z TYPE=cluster-admin-webpanel BAD=5 URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=1 QGA=active BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-webpanel.txt","sha256":"0f08c7d468ae24f3d00ee62b8294a7f27b6a37d7c5f5b4ed74b38734d7ba2cd8"},{"bytes":199,"matches":[{"line":1,"text":"STATUS="}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-incident-engine-web.txt","sha256":"bd78aefe2cd3a9afdfbbecca931c19e5a9dcb8b725ef083aaffd181fc0ec0680"},{"bytes":272,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T12:53:26Z TYPE=cluster-admin-observer BAD=2 WARN=0 HOST=cluster-admin SELF_FAILED_UNITS=1 QGA=active PVE01=checked PVE02=checked PVE03=checked EDGE=checked DNS_PAIR=checked OVERALL_ENDPOINT=checked REPORT=/var/lib/homelab-cluster-admin/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/cluster-admin-observer.txt","sha256":"3c2114bac576dbe11c26a843f19223c677a91d7fd19b25eadf6fd3f0b9da47ec"},{"bytes":380,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T04:49:23Z TYPE=cluster-admin-strict-final-seal BAD=0 VMID=180 IP=[PRIVATE_IP] MODE=DEDICATED_CLUSTER_ADMIN_HARDGATED OBSERVER=OK RESTRICTED_PROBES=OK FULL_OBSERVER=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-strict-final-seal-latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt","sha256":"03a07c2be6dd4900c328c05aded594879117efbc02a8d2a7e89ae2d9847dfe8e"},{"bytes":1535,"matches":[{"line":1,"text":"STATUS="}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/overall.txt","sha256":"0ec7f8bcad6d2578f209b222be1135e7fa03f801e1e07bdbb634ba5f536a101b"},{"bytes":128,"matches":[{"line":1,"text":"STATUS=WARN TS=2026-08-05T05:21:50Z TYPE=final-readiness BAD=6 PASS=47 FAIL=6 REPORT=/var/lib/homelab-final-readiness/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/final-readiness.txt","sha256":"a15ee89bc9826b84a5b52e81f63a133b127bd7e263223eabf6b9ec62930e35ac"},{"bytes":217,"matches":[{"line":1,"text":"STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md"}],"mode":"0o644","path":"/var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt","sha256":"e03b236f41f8d66f070ecd3234b3ff0836a44d972fd6f6abc721b9683fa22053"}],"process_lines":["    439       1 root     root     python3         /usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin","    487       1 cluster+ cluster+ python3         /usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1"],"proxies":[],"pve01_pushed_reference_files":[{"bytes":3708,"matches":[{"line":4,"text":"WEB=/var/www/homelab-cluster-admin"},{"line":13,"text":"SELF=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-self-check.txt)\""},{"line":14,"text":"OBSERVER=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-observer.txt)\""},{"line":15,"text":"FULL=\"$(line_or_missing /var/lib/homelab-health/cluster-admin-full-observer.txt)\""},{"line":16,"text":"BACKUP=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-vm-mail-cloud-backup.txt)\""},{"line":17,"text":"OVERALL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/overall.txt)\""},{"line":18,"text":"FINAL=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/final-readiness.txt)\""},{"line":19,"text":"STRICT=\"$(line_or_missing /var/lib/homelab-health/pve01-pushed/cluster-admin-strict-final-seal.txt)\""},{"line":30,"text":"# cycle-break: pushed overall and final-readiness are display context, not webpanel hard gates"},{"line":35,"text":"STATUS_LINE=\"STATUS=$STATUS TS=$TS TYPE=cluster-admin-webpanel BAD=$BAD URL=http://[PRIVATE_IP]:8080/ STATUS_URL=http://[PRIVATE_IP]:8080/status.txt SELF_FAILED_UNITS=$FAILED QGA=$QGA BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK\""},{"line":36,"text":"echo \"$STATUS_LINE\" | tee /var/lib/homelab-health/cluster-admin-webpanel.txt >/dev/null"},{"line":46,"text":"Homelab Cluster Admin"},{"line":58,"text":"
Homelab Cluster Admin
"},{"line":74,"text":"
Service map tail
$(tail -80 /var/lib/homelab-cluster-admin/full-observer-latest.md 2>/dev/null)
"},{"line":79,"text":"cat /var/lib/homelab-health/cluster-admin-webpanel.txt"}],"mode":"0o755","path":"/opt/homelab-cluster-admin/render-webpanel.sh","sha256":"fbec0df3b65d3e9e00348441b018382a603db18469d07cde13a492c09cb0790f"}],"scan":{"directories_scanned":68,"index_count":1,"match_count":42},"units":[{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001780 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-incident-engine-collector.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.service\n[Unit]\nDescription=Cluster Admin inventory metrics and incident collector\nAfter=network-online.target postgresql.service cluster-admin-incident-engine.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nExecStart=/usr/bin/python3 /opt/cluster-admin-incident-engine/collector.py\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine /var/lib/homelab-health"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine-collector.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-incident-engine-collector.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-incident-engine-collector.timer","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine-collector.timer\n[Unit]\nDescription=Run Cluster Admin incident collector every minute\n\n[Timer]\nOnBootSec=30s\nOnUnitActiveSec=60s\nAccuracySec=10s\nPersistent=true\nUnit=cluster-admin-incident-engine-collector.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","EnvironmentFiles":"/etc/cluster-admin-incident-engine/runtime.conf (ignore_errors=no)","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1 ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:07 UTC] ; stop_time=[n/a] ; pid=487 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/cluster-admin-incident-engine.service","Group":"clusteradmin","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"clusteradmin","WorkingDirectory":"/opt/cluster-admin-incident-engine"},"show_rc":0,"unit":"cluster-admin-incident-engine.service","unit_text":"# /etc/systemd/system/cluster-admin-incident-engine.service\n[Unit]\nDescription=Cluster Admin Incident Engine web application\nAfter=network-online.target postgresql.service\nWants=network-online.target\nRequires=postgresql.service\n\n[Service]\nType=simple\nUser=clusteradmin\nGroup=clusteradmin\nWorkingDirectory=/opt/cluster-admin-incident-engine\nEnvironmentFile=/etc/cluster-admin-incident-engine/runtime.conf\nExecStart=/usr/bin/python3 -m uvicorn app:app --host 0.0.0.0 --port 8081 --proxy-headers --forwarded-allow-ips=[PRIVATE_IP],127.0.0.1\nRestart=on-failure\nRestartSec=5\nUMask=0027\nNoNewPrivileges=true\nPrivateTmp=true\nProtectSystem=strict\nProtectHome=true\nReadOnlyPaths=/etc/cluster-admin-incident-engine\nReadWritePaths=/var/lib/cluster-admin-incident-engine\nRestrictSUIDSGID=true\nLockPersonality=true\nRestrictRealtime=true\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:54:57 UTC] ; stop_time=[Wed 2026-08-05 12:54:58 UTC] ; pid=2001764 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.service","Group":"clusteradmin","LoadState":"loaded","SubState":"dead","TriggeredBy":"cluster-admin-probe-agent.timer","Triggers":"","UnitFileState":"static","User":"clusteradmin","WorkingDirectory":""},"show_rc":0,"unit":"cluster-admin-probe-agent.service","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.service\n[Unit]\nDescription=Cluster Admin restricted read-only probe controller\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUser=clusteradmin\nGroup=clusteradmin\nEnvironment=CLUSTER_ADMIN_PROBE_HEALTH=/var/lib/cluster-admin-probe-agent/cluster-admin-probe-agent.txt\nExecStart=/usr/bin/python3 /opt/cluster-admin-probe-agent/controller.py --run\nUMask=0077\nNoNewPrivileges=yes\nPrivateTmp=yes\nPrivateDevices=yes\nProtectSystem=strict\nProtectHome=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectClock=yes\nProtectHostname=yes\nLockPersonality=yes\nMemoryDenyWriteExecute=yes\nRestrictSUIDSGID=yes\nRestrictRealtime=yes\nRemoveIPC=yes\nCapabilityBoundingSet=\nAmbientCapabilities=\nRestrictAddressFamilies=AF_UNIX AF_INET AF_INET6\nIPAddressDeny=any\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nIPAddressAllow=[PRIVATE_IP]\nReadOnlyPaths=/etc/cluster-admin-probe-agent /opt/cluster-admin-probe-agent\nReadWritePaths=/var/lib/cluster-admin-probe-agent\nStateDirectory=cluster-admin-probe-agent\nStateDirectoryMode=0750\nRuntimeDirectory=cluster-admin-probe-agent\nTimeoutStartSec=70"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/cluster-admin-probe-agent.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"cluster-admin-probe-agent.service","UnitFileState":"enabled"},"show_rc":0,"unit":"cluster-admin-probe-agent.timer","unit_text":"# /etc/systemd/system/cluster-admin-probe-agent.timer\n[Unit]\nDescription=Run Cluster Admin restricted probes every minute\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=60s\nRandomizedDelaySec=5s\nPersistent=true\nUnit=cluster-admin-probe-agent.service\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/full-observer.sh ; argv[]=/opt/homelab-cluster-admin/full-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:33 UTC] ; pid=2001289 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-full-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.service\n[Unit]\nDescription=Homelab cluster-admin full observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/full-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-full-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-full-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-full-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-full-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin full observer\n\n[Timer]\nOnBootSec=5min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/cluster-observer.sh ; argv[]=/opt/homelab-cluster-admin/cluster-observer.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:27 UTC] ; pid=2001290 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-observer.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-observer.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.service\n[Unit]\nDescription=Homelab cluster-admin observer\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/cluster-observer.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-observer.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-observer.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-observer.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-observer.timer\n[Unit]\nDescription=Run homelab cluster-admin observer\n\n[Timer]\nOnBootSec=3min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"","ExecStart":"{ path=/opt/homelab-cluster-admin/self-check.sh ; argv[]=/opt/homelab-cluster-admin/self-check.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:53:26 UTC] ; stop_time=[Wed 2026-08-05 12:53:32 UTC] ; pid=2001296 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-self-check.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-self-check.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.service\n[Unit]\nDescription=Homelab cluster-admin self check\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/self-check.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-self-check.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-self-check.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-self-check.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-self-check.timer\n[Unit]\nDescription=Run homelab cluster-admin self check\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=5min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"inactive","DropInPaths":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf","ExecStart":"{ path=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; argv[]=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:55:27 UTC] ; stop_time=[Wed 2026-08-05 12:55:28 UTC] ; pid=2001781 ; code=exited ; status=0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.service","Group":"","LoadState":"loaded","SubState":"dead","TriggeredBy":"homelab-cluster-admin-webpanel-render.timer","Triggers":"","UnitFileState":"static","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service\n[Unit]\nDescription=Render Homelab cluster-admin web panel\n\n[Service]\nType=oneshot\nExecStart=/opt/homelab-cluster-admin/render-webpanel.sh\n\n# /etc/systemd/system/homelab-cluster-admin-webpanel-render.service.d/10-cycle-break.conf\n[Service]\nExecStart=\nExecStart=/opt/homelab-cluster-admin/render-webpanel-cycle-break.sh"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel-render.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-cluster-admin-webpanel-render.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-cluster-admin-webpanel-render.timer","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel-render.timer\n[Unit]\nDescription=Refresh Homelab cluster-admin web panel\n\n[Timer]\nOnBootSec=1min\nOnUnitActiveSec=1min\nPersistent=true\n\n[Install]\nWantedBy=timers.target"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","ExecStart":"{ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin ; ignore_errors=no ; start_time=[Tue 2026-07-28 11:06:03 UTC] ; stop_time=[n/a] ; pid=439 ; code=(null) ; status=0/0 }","FragmentPath":"/etc/systemd/system/homelab-cluster-admin-webpanel.service","Group":"","LoadState":"loaded","SubState":"running","TriggeredBy":"","Triggers":"","UnitFileState":"enabled","User":"","WorkingDirectory":""},"show_rc":0,"unit":"homelab-cluster-admin-webpanel.service","unit_text":"# /etc/systemd/system/homelab-cluster-admin-webpanel.service\n[Unit]\nDescription=Homelab cluster-admin web panel\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 -m http.server 8080 --bind 0.0.0.0 --directory /var/www/homelab-cluster-admin\nRestart=always\nRestartSec=5\n\n[Install]\nWantedBy=multi-user.target"},{"cat_rc":0,"show":{"ActiveState":"failed","DropInPaths":"","ExecStart":"{ path=/usr/local/sbin/homelab-stage4i-verify ; argv[]=/usr/local/sbin/homelab-stage4i-verify ; ignore_errors=no ; start_time=[Wed 2026-08-05 12:41:27 UTC] ; stop_time=[Wed 2026-08-05 12:41:27 UTC] ; pid=1999359 ; code=exited ; status=50 }","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.service","Group":"root","LoadState":"loaded","SubState":"failed","TriggeredBy":"homelab-stage4i-verify.timer","Triggers":"","UnitFileState":"static","User":"root","WorkingDirectory":""},"show_rc":0,"unit":"homelab-stage4i-verify.service","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.service\n[Unit]\nDescription=Verify active immutable Stage 4I task version\nAfter=local-fs.target\n\n[Service]\nType=oneshot\nUser=root\nGroup=root\nExecStart=/usr/local/sbin/homelab-stage4i-verify\nNoNewPrivileges=true\nPrivateDevices=true\nProtectClock=true\nProtectControlGroups=true\nProtectHome=true\nProtectKernelLogs=true\nProtectKernelModules=true\nProtectKernelTunables=true\nRestrictNamespaces=true\nRestrictRealtime=true\nLockPersonality=true\nMemoryDenyWriteExecute=true\nUMask=0027"},{"cat_rc":0,"show":{"ActiveState":"active","DropInPaths":"","FragmentPath":"/etc/systemd/system/homelab-stage4i-verify.timer","LoadState":"loaded","SubState":"waiting","TriggeredBy":"","Triggers":"homelab-stage4i-verify.service","UnitFileState":"enabled"},"show_rc":0,"unit":"homelab-stage4i-verify.timer","unit_text":"# /etc/systemd/system/homelab-stage4i-verify.timer\n[Unit]\nDescription=Regularly verify active immutable Stage 4I task version\n\n[Timer]\nOnBootSec=2min\nOnUnitActiveSec=15min\nAccuracySec=1min\nPersistent=true\nUnit=homelab-stage4i-verify.service\n\n[Install]\nWantedBy=timers.target"}],"webroots":[]},"remote_target":{"evidence":[{"line":21,"target_count":1,"text":"scp -q -o BatchMode=yes -o ConnectTimeout=8 \"$TMP\"/*.txt debian@:/tmp/"},{"line":24,"target_count":1,"text":"ssh -n -o BatchMode=yes -o ConnectTimeout=8 debian@ 'sudo install -d -m 0755 /var/lib/homelab-health/pve01-pushed; for f in /tmp/cluster-admin-vm-mail-cloud-backup.txt /tmp/overall.txt /tmp/final-readiness.txt /tmp/cluster-admin-strict-final-seal.txt; do test -s \"$f\" && sudo install -m 0644 \"$f\" /var/lib/homelab-health/pve01-pushed/; done'"}],"target_cardinality":1,"target_host_is_private_ipv4":true,"target_host_sha256":"7f25d5297f515cb92000161cbee4cb49a7fbbbd1c0ef2466a5d88644f4bb6d69","target_user":"debian"},"rollback_restored":false,"rollback_started":false,"schema":15,"status":"OK","transaction_gate":{"application_architecture_found":true,"next_required_action":"build one corrected local+remote atomic transaction","pushed_health_reference_found":true,"ready_for_corrected_atomic_transaction":true,"remote_probe_root":true},"transport":{"attempts":[{"privilege":"sudo-root","rc":0,"stderr":"","stderr_bytes":0,"stderr_sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stdout_bytes":82754,"stdout_sha256":"603f5eb85c971d0bc029287cf91d9d0239b132dd409df5bbf8c5f47a7bb32738"}],"privilege":"sudo-root","success":true},"v14_exact_result":{"changes_made":false,"control_plane_error":null,"identity_confirmed":true,"mutation_outcome":"NO_MUTATION","output_bytes":10768,"output_sha256":"cc30d7f4e6be0ab5e78545d4537a39178b59e9f1a79227efaec134b78f6fc233","rollback_restored":null,"rollback_started":false,"runner_document_rc":0,"runner_json_sha256":"c932bbad4dd7f176e29621f3c549e2571baf7f84500ef2c5408a04c6a84083ff","runner_rc":0,"runner_status":"OK","runner_text_sha256":"70796c2456365c9224dd6ff02897d29412349eb1a5fea03c89ae9212217a03f8","runner_wrapper_matches_document":true,"state_document_sha256":"f3db8a869123d1266f906fdd02ca18a890141c2bd63e2bc57a3a0e7c5962f319","state_path_sha256":"2c050c8423a48aab8f448c94123e67f132381be3232e1d10274f4e87e1013b16","state_status":"OK"}}
+HOMELAB_RESULT_CONTRACT={"changes_made":false,"command_id":"HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z","command_rc":0,"mutation_outcome":"NO_MUTATION","output_truncated_in_json":false,"rc":0,"rollback_restored":null,"rollback_started":false,"status":"OK","version":1}
+{"bounded_adapter_scan":null,"changes_made":false,"command_id":"HOMELAB-CLUSTER-KB-V1-EXACT-OUTPUT-PROJECTION-READ-ONLY-20260805T130500Z","command_rc":0,"evidence_bundle":{"bundle_bytes_ok":true,"bundle_sha256_ok":true,"exists":true,"member_count":6,"output_member_matches":0,"path_is_exact_from_previous_result":true,"runner_json_member_matches":1,"runner_text_member_matches":1,"unsafe_member_rejected":false},"files_written":false,"mutation_outcome":"NO_MUTATION","network_requests_made":false,"next_action":"RUN_ONE_READ_ONLY_GITEA_DEFINED_CLUSTER_FACT_AND_DOCUMENT_GAP_ANALYSIS","output_truncated_in_json":false,"pre_command_known_error_gate":{"checked":true,"count":28,"ids":["candidate-file-limit-600-20260805","runner-wrapper-mismatch-after-success-20260805","canonical-cluster-handbook-ambiguous-20260805","pvesh-adapter-context-unreadable-20260805","homelab-ops-bare-repository-not-found-local-20260805","gitea-network-fallback-stage-not-emitted-20260805","gitea-owner-hardcoded-homelab-admin-20260805","gitea-git-auth-required-or-credential-unavailable-20260805","gitea-process-self-match-payload-name-20260805","sanitize-bool-attributeerror-20260805","reader-v3-path-misclassified-as-route-20260805","reader-v3-loopback-assumption-20260805","reader-v3-capability-not-treated-as-secret-20260805","reader-v3-systemd-unit-fed-to-python-ast-20260805","reader-v3-helper-control-flow-not-traced-20260805","reader-v3-post-capability-operation-not-derived-20260805","reader-v3-detected-operations-not-tested-20260805","reader-v3-manifest-global-not-bound-in-sandbox-20260805","reader-v3-result-file-second-source-not-read-20260805","reader-v3-negative-control-mixed-with-production-20260805","reader-v3-manifest-schema-guessed-20260805","reader-result-output-bound-after-compaction-20260805","reader-v3-generation-record-shape-guessed-20260805","reader-v3-result-file-assumed-standalone-json-20260805","reader-v3-gitea-detail-marker-false-inventory-20260805","reverse-proxy-runtime-not-visible-on-host-20260805","gitea-public-owner-not-anonymously-discoverable-20260805","exact-output-long-line-not-projectable-by-portal-view-20260805"]},"projection":{"analysis_outcome":{"error_fingerprint":"26d65952ac8f64197a6fd76ec56e3b5ffda8c4838efe2af2947a603d01270b92","next_action":"RUN_ONE_READ_ONLY_GITEA_DEFINED_CLUSTER_FACT_AND_DOCUMENT_GAP_ANALYSIS","ready_for_gitea_defined_cluster_fact_analysis":true,"root_cause_classification":"GITEA_CLUSTER_HANDBOOK_AND_ERROR_SYSTEM_DEEPLY_ANALYZED","status":"OK"},"canonical_handbook":{"candidates":[{"canonical_phrase_hits":3,"canonical_score":227,"content_sha256":"8051e3e761493a1b7088a0f3ed7e0aa9218673f453ca88171178364d26701f64","headings":[],"inbound_reference_count":8,"path":"errors/regression-cases.json"},{"canonical_phrase_hits":0,"canonical_score":74,"content_sha256":"a49bcb32ac7212d7fb9768db8a277cb5525c873c84dd6fb633edae35dc6de532","headings":[],"inbound_reference_count":0,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh"},{"canonical_phrase_hits":1,"canonical_score":56,"content_sha256":"ac5dc881dcd9fcb72644bc893e866f463c9c2b9b4544e32434e21b5d34168fb5","headings":[],"inbound_reference_count":1,"path":"inventory/access-paths.json"},{"canonical_phrase_hits":0,"canonical_score":55,"content_sha256":"666072447ba11d346772f62f274dadc974c4463e20f67e67f82f228f1fcf2ce8","headings":[],"inbound_reference_count":0,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/phases/rollback.sh"},{"canonical_phrase_hits":0,"canonical_score":54,"content_sha256":"636254d2526f0f62184d47cdb4c30c2831b11869c2000c557666f60b55d7691b","headings":[],"inbound_reference_count":0,"path":"tasks/cluster-cloud-quorum-vm160-canary-v1/task.json"}],"path":"errors/regression-cases.json","status":"UNIQUE_CONFIRMED"},"documentation_coverage":{"access":{"file_count":22,"paths":["changes/CR-2026-0018/regression-cases-draft.json","docs/planned/skladchik-moderator-assistant.md","errors/regression-cases.json","inventory/access-paths.json","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/dry-run.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/prepare.sh"]},"backup":{"file_count":29,"paths":["docs/ARCHITECTURE.md","docs/skladchik-cookie-refresh.md","errors/regression-cases.json","observed/cluster-guests.json","policies/cluster-cloud-quorum.json","schemas/change.schema.json","tasks/cluster-cloud-quorum-vm160-canary-v1/assets/pve02-vm160-preflight-probe.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/assets/vm160-cloud-quorum-worker.sh"]},"error_system":{"file_count":44,"paths":[".gitea/workflows/ci.yml","changes/CR-2026-0018/caller-inventory-requirements.md","changes/CR-2026-0018/regression-cases-draft.json","changes/README.md","docs/ARCHITECTURE.md","docs/skladchik-cookie-refresh.md","docs/WORKFLOW.md","errors/regression-cases.json"]},"internal_external":{"file_count":7,"paths":["changes/CR-2026-0018/regression-cases-draft.json","errors/regression-cases.json","policies/operating-policy.json","tasks/homelab-logrotate-namespace-latch-clearance-v1/README.txt","tests/test_cr_2026_0005.py","tests/test_cr_2026_0011_skladchik_cookie_jar_reset.py","tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py"]},"repair":{"file_count":31,"paths":["changes/CR-2026-0018/caller-inventory-requirements.md","changes/README.md","docs/ARCHITECTURE.md","docs/skladchik-cookie-refresh.md","docs/WORKFLOW.md","errors/regression-cases.json","policies/cluster-cloud-quorum.json","policies/operating-policy.json"]},"services":{"file_count":21,"paths":["changes/CR-2026-0018/caller-inventory-requirements.md","docs/ARCHITECTURE.md","docs/planned/skladchik-moderator-assistant.md","errors/regression-cases.json","observed/cluster-guests.json","tasks/cluster-cloud-quorum-vm160-canary-v1/assets/vm160-cloud-quorum-worker.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/apply.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/dry-run.sh"]}},"enumeration":{"candidate_file_count":50,"fetch_failure_count":0,"fetched_file_count":50,"fetched_total_bytes":173536,"tree_blob_count":125,"tree_complete":true,"tree_directory_count":36,"tree_entry_count":161},"error_system_current":{"duplicate_error_ids":[],"duplicate_fingerprints":[],"enforcement_path_count":7,"enforcement_paths":["schemas/change.schema.json","schemas/context.schema.json","tasks/cluster-cloud-quorum-vm160-canary-v1/assets/pve02-vm160-preflight-probe.sh","tasks/cluster-cloud-quorum-vm160-canary-v1/phases/preflight.sh","tasks/homelab-logrotate-namespace-latch-clearance-v1/phases/preflight.sh","tasks/skladchik-reports-monitor-controlled-reauth-v1/assets/edge-preflight.sh","tasks/skladchik-reports-monitor-controlled-reauth-v1/phases/preflight.sh"],"gate_signals":{"ci_workflow_present":true,"duplicate_failed_command_block_signal":false,"error_register_checked_signal":false,"fingerprint_signal":false,"preflight_references_error_register":true,"reference_register_checked_signal":false,"schema_validation_signal":true,"tests_present":true},"machine_parseable_registry_file_count":2,"prevention_evidence_count":0,"registry_path_count":2,"registry_paths":["changes/CR-2026-0018/regression-cases-draft.json","errors/regression-cases.json"],"regression_path_count":2,"regression_paths":["changes/CR-2026-0018/regression-cases-draft.json","errors/regression-cases.json"],"superseded_path_count":0,"superseded_paths":[],"test_path_count":14,"test_paths":["tests/fixtures/gitea-legacy-auth-permission.json","tests/test_cr_2026_0005.py","tests/test_cr_2026_0006.py","tests/test_cr_2026_0006_vm160_task.py","tests/test_cr_2026_0007_skladchik_reauth.py","tests/test_cr_2026_0007_skladchik_semantic_repair.py","tests/test_cr_2026_0008_skladchik_snapshot_integrity.py","tests/test_cr_2026_0009_skladchik_concurrency_guard.py","tests/test_cr_2026_0011_skladchik_cookie_jar_reset.py","tests/test_cr_2026_0012_skladchik_cookie_refresh_runbook.py"],"unique_error_id_count":61,"unique_fingerprint_count":0,"workflow_path_count":1,"workflow_paths":[".gitea/workflows/ci.yml"]},"gaps":["no_static_evidence_error_fingerprint_gate"],"pre_command_gate":{"checked":true,"projection_command_known_error_count":28,"regression_tests_passed":true,"reported_error_count":27},"previous_exact_result":{"capsule_sha256":"031aa6c1029c24d8b7dac8192707376fa67ee93cdcfceedcfb768a0b7d4ceb8e","command_id":"HOMELAB-CLUSTER-KB-V1-GITEA-REPOSITORY-CONTENT-DEEP-ANALYSIS-READ-ONLY-20260805T124500Z","identity_fully_verified":true,"output_bytes":27219,"output_sha256":"ead28fd156e646c27f6e988c12ea1e89a498882f07071faacba670e87c8a7983","runner_json_sha256":"e3211267d27f7497339cd34ecdbe0cce21d039965625ffd5f3236dc20513b082","runner_text_sha256":"083286377044254ff016f0f9024655e228a67ae2737d7e1d9641996d5af8e09a"},"previous_result_contract":{"changes_made":false,"command_rc":0,"mutation_outcome":"NO_MUTATION","output_truncated_in_json":false,"rc":0,"rollback_restored":null,"rollback_started":false,"status":"OK"},"recommended_error_system_v2":{"acceptance_criteria":["zero hard-coded owner/path/host/version unless proven by exact prior evidence or current Gitea content","zero mutation without byte-for-byte backup and tested rollback","zero repeated known error fingerprint","all command outputs bounded and non-truncated","all new errors recorded before any subsequent command"],"components":[{"component":"machine_error_registry","requirement":"one append-only machine-readable record per proven failure with stable id, normalized fingerprint, exact command/capsule/result SHA, root cause, correction and prevention test"},{"component":"pre_command_gate","requirement":"before every .hcmd, read canonical cluster reference and entire active error registry; block duplicate failed fingerprints and commands that violate a prevention test"},{"component":"transaction_manifest","requirement":"every mutation declares exact source SHA, candidate SHA, byte backup SHA, preconditions, atomic apply, verification and deterministic rollback"},{"component":"post_result_ingestion","requirement":"after exact-result verification, automatically classify the outcome, append new proven failures, close corrected cases and link superseded commands"},{"component":"regression_suite","requirement":"positive and negative fixtures for every recorded error; payload self-test and repository CI must pass before command publication"},{"component":"knowledge_freshness","requirement":"canonical handbook stores last verified fact source, verification time, source SHA and stale/unknown markers; commands may not rely on stale or unknown values"}],"grounded_in_detected_gaps":["no_static_evidence_error_fingerprint_gate"],"minimum_gate_order":["verify previous exact result identity","read canonical handbook at exact branch commit","read active errors and prevention tests","validate target facts from authoritative source","run payload regression tests","only then publish or execute the next command"],"name":"Error System V2: evidence-first command gate","reuse_existing_error_root":"changes/CR-2026-0018"},"repository":{"branch_commit_sha":"fe1221b06643db3b00a621b0230f92a46a4edc2b","default_branch":"main","full_name":"homelab-admin/homelab-ops","private":true},"source_result_projection":{"budget_bytes":29232,"bytes":26771,"max_bytes":30000,"steps":["top_analyzed_files_12","handbook_candidates_6"]}},"projection_bytes_before_final_wrapper":11730,"rc":0,"read_only":true,"regression_tests":{"duplicate_sha_string_rejected":true,"passed":true,"recursive_command_identity":true,"unique_sha_string_extraction":true},"rollback_restored":null,"rollback_started":false,"root_cause_classification":"EXACT_PRIOR_GITEA_DEEP_ANALYSIS_OUTPUT_PROJECTED","schema":1,"status":"OK"}
 
 OUTPUT_END
 CHAT_OUTPUT_END