diff --git a/runtime/history/CONTEXT-SOURCES-FULL-REFRESH-363.json b/runtime/history/CONTEXT-SOURCES-FULL-REFRESH-363.json new file mode 100644 index 0000000..9f7d665 --- /dev/null +++ b/runtime/history/CONTEXT-SOURCES-FULL-REFRESH-363.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "CONTEXT-SOURCES-FULL-REFRESH-363", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-context", + "started_at_utc": "2026-07-23T14:23:43Z", + "finished_at_utc": "2026-07-23T14:23:43Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "eb33834ec5df95efdbe55c4264ce02cbc69fcf90de62d457948a23d1e1efda04", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "9482ef04e0765ed01d6de068ebba4ef30fb0e133871b6cd45c4af4906d299122", + "sanitized_output_sha256": "3d9b0a662d873becba40d548b6854862d5ad518285238a73362573ff85988751", + "output_truncated_in_json": true, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/test\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/mktemp\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/rm\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/curl\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/jq\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/stat\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/sha256sum\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/awk\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/date\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/cat\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/printf\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/grep\nDETAIL=STAGE=SOURCE_FETCH\nDETAIL=IMMUTABLE_SOURCE=errors-index|SIZE=146628|SHA256=a13ca59d5fa1ad39b813bad3115d7b8cf8c5c68935fc813dbd792757ed421e24|STATUS=PASS\nDETAIL=IMMUTABLE_SOURCE=workflow-contract|SIZE=1926|SHA256=1378a2c0d6dec6b09e3ea7dfced65fed1e1c38065baa6bba8bfd4c89e23e2c78|STATUS=PASS\nDETAIL=IMMUTABLE_SOURCE=double-check-rule|SIZE=268|SHA256=7e5221154959bb0f435b3cfb955e6671d3c361ecc7a99625f04252cc74ebaa4b|STATUS=PASS\nDETAIL=FULL_CONTEXT_BUNDLE_BEGIN\nFULL_CONTEXT_BUNDLE_SCHEMA=1\nGENERATED_AT_UTC=2026-07-23T14:23:43Z\nREFERENCE_FILE=/etc/pve/31_HOMELAB_REFERENCE.md\nREFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e\nREFERENCE_SIZE=164602\nERROR_REGISTER_FILE=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\nERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752\nERROR_REGISTER_SIZE=79246\nAUTO_CONTEXT_ID=CONTEXT-AUTO-20260723T141652Z\nAUTO_CONTEXT_COMMIT=fcf2ed2a6b11016ec6441a014f7832a86c51f793\nAUTO_CONTEXT_URL=https://git.gram1.ru/homelab-admin/homelab-public-context/raw/commit/fcf2ed2a6b11016ec6441a014f7832a86c51f793/runtime/history/CONTEXT-AUTO-20260723T141652Z.txt\nAUTO_CONTEXT_SHA256=acd6a206ec5eb45ed50c2aae1cee02d4683dd8675e5c2d88190cefcd2446dada\nAUTO_CONTEXT_SIZE=29049\nLATEST_JSON_AUTHORITATIVE=NO\nDETAIL=SECTION_REFERENCE_REGISTER_BEGIN\nHOMELAB REFERENCE\nGENERATED=2026-06-29T21:45:29+03:00\nAUDIT_DIR=/root/cluster-audit-20260629T201245\n\nCORE_CLUSTER_CONFIG\n\nkeyboard: en-us\nmigration: secure,network=[PRIVATE_IP]/24\n\nFINAL_CLASSIFICATION\nMISSING_EXPECTED_PREFIX_COUNT=0\nSTRICT_POSSIBLE_SECRET_COUNT=0\nSTRICT_SECRET_SCAN_OK\nHEALTH_WARN_ERROR_COUNT=14\nNOTE exclude_from_reference_or_delete_later=05_edge_compose_safe.tgz\nNOTE forum-prod VMID160 is not in homelab-nightly-all; it has separate local/cloud backup chain\nNOTE edge-vm disk scsi1 backup=0 risk must be documented\nNOTE Corosync remains on [PRIVATE_IP]/12/13; migration network is [PRIVATE_IP]/24 secure\nNOTE edge health WARN/ERROR items should be documented as known current states\n\nHEALTH_NOT_OK\nHEALTH_NOT_OK item=adguard-rewrite-sync.txt line=STATUS=ERROR TS=20260629T152553Z TYPE=adguard-rewrite-sync REASON=URLError SECRET_PRINTED=REDACTED\nHEALTH_NOT_OK item=backup-restore-coverage-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=backup-restore-dashboard.txt line=STATUS=WARN TS=2026-06-29T18:10:09Z ITEMS=80 NOT_OK=13\nHEALTH_NOT_OK item=container-image-lifecycle-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=cron-job-monitoring-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=direct-heartbeat-pilot-readiness.txt line=STATUS=WARN\nHEALTH_NOT_OK item=healthchecks-heartbeat-coverage.txt line=STATUS=WARN\nHEALTH_NOT_OK item=healthchecks-job-coverage.txt line=STATUS=WARN\nHEALTH_NOT_OK item=healthchecks-job-monitoring-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=homepage-service-catalog-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=ingress-dns-route-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=job-monitoring-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=observability-health-surface-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=security-vulnerability-slo.txt line=STATUS=WARN\n\nEVIDENCE_FILES\n00_cluster_overview.txt 3544 bytes\n00_HOMELAB_ASSISTANT_ERROR_REGISTER.md 2566 bytes\n01_pve01_audit.txt 28554 bytes\n02_pve02_audit.txt 24786 bytes\n03_pve03_audit.txt 16842 bytes\n04_edge_vm_basic.txt 34506 bytes\n05_edge_compose_redacted.txt 24228 bytes\n05_edge_compose_safe.tgz.QUARANTINED.txt 354 bytes\n05_edge_compose_tar_errors.txt 166 bytes\n06_ACCESS_AND_ERROR_RULES.md 13742 bytes\n06_edge_npmplus_routes_safe.txt 17350 bytes\n07_edge_systemd_backup_audit.txt 20694 bytes\n08_edge_scripts_redacted.txt 198622 bytes\n09_forum_prod_basic.txt 14413 bytes\n10_forum_codevipe_xenforo_audit.txt 3861 bytes\n11_forum_backup_audit_redacted.txt 5952 bytes\n12_pve02_codevipe_cloud_backup_audit.txt 6277 bytes\n13_pve01_systemd_backup_audit.txt 60109 bytes\n14_pve01_scripts_redacted.txt 246474 bytes\n15_pve01_ct_110_audit.txt 5049 bytes\n15_pve01_ct_112_audit.txt 4843 bytes\n16_pve02_ct_111_audit.txt 4758 bytes\n16_pve02_ct_113_audit.txt 4843 bytes\n17_nextcloud_vm_audit.txt 13293 bytes\n18_nextcloud_backup_restore_pve01_audit.txt 4269 bytes\n19_pve02_host_automation_audit.txt 10642 bytes\n19_pve03_host_automation_audit.txt 7632 bytes\n20_AUDIT_INDEX_AND_SECRET_SCAN.txt 11935 bytes\n21_proxmox_cluster_config_audit.txt 10855 bytes\n22_internal_2_5g_network_inventory.txt 8036 bytes\n23_cluster_internal_network_configured.txt 747 bytes\n24_cluster_internal_network_speedtest.txt 54665 bytes\n25_cluster_internal_network_migration_enabled.txt 1816 bytes\n26_migration_network_pvesh_verify.txt 1210 bytes\n27_migration_network_canonical_verify.txt 956 bytes\n28_dns_configs_redacted.txt 15559 bytes\n29_health_summary_all_nodes.txt 39051 bytes\n30_FINAL_AUDIT_GAPS_WARNINGS_SECRET_CLASSIFICATION.txt 4366 bytes\n31_HOMELAB_REFERENCE.md 1805 bytes\ndatacenter.cfg.before-canonical-migration-20260629T211046 63 bytes\ndatacenter.cfg.before-migration-network-20260629T210710 16 bytes\n\n\n\nРУССКОЕ_РЕЗЮМЕ_КЛАСТЕРА\n- Кластер: homelab, 3 узла, quorum OK.\n- LAN: pve01=[PRIVATE_IP], pve02=[PRIVATE_IP], pve03=[PRIVATE_IP].\n- Внутренняя сеть: pve01-int=[PRIVATE_IP], pve02-int=[PRIVATE_IP], pve03-int=[PRIVATE_IP].\n- Миграции Proxmox: migration: secure,network=[PRIVATE_IP]/24.\n- Corosync остаётся на [PRIVATE_IP]/12/13; не переносить без отдельного rollback-плана.\n\nРИСКИ_И_ДЕЙСТВИЯ\n- VM160 forum-prod не входит в ночной Proxmox backup.\n- У VM130 edge-vm есть риск: дополнительный диск backup=0.\n- Нужно ротировать ранее засвеченный Cloudflare token.\n- Желательно ротировать XenForo SMTP пароль, который был выведен до редактирования.\n- Проверить adguard-rewrite-sync: STATUS=ERROR, REASON=URLError.\n- Проверить backup-restore-dashboard: STATUS=WARN, NOT_OK=13.\n\nДОСТУПЫ_КРАТКО\n- Proxmox: ssh root@pve01, ssh root@pve02, ssh root@pve03.\n- Edge VM: ssh debian@[PRIVATE_IP], использовать sudo, root-login не использовать.\n- Nextcloud VM: ssh debian@[PRIVATE_IP].\n- Forum-prod: сначала ssh root@pve02, затем ssh -i [SENSITIVE_PATH] root@[PRIVATE_IP].\n\nНАГРУЗКИ_И_СЕРВИСЫ\n- CT110 dns1 pve01 [PRIVATE_IP] AdGuard Home.\n- CT111 dns2 pve02 [PRIVATE_IP] AdGuard Home.\n- CT112 unbound1 pve01 [PRIVATE_IP] Unbound.\n- CT113 unbound2 pve02 [PRIVATE_IP] Unbound.\n- VM130 edge-vm pve03 [PRIVATE_IP] Docker ingress/app host.\n- VM150 Nextcloud pve01 [PRIVATE_IP] Nextcloud AIO.\n- VM160 forum-prod pve02 [PRIVATE_IP] CodeVipe/XenForo.\n\nBACKUP_КРАТКО\n- Ночной Proxmox backup включает VMID 110,111,112,113,130,150.\n- VM160 forum-prod исключён из ночного Proxmox backup и держится на app-level local/cloud backup.\n- Edge VM имеет vzdump/offhost/restore evidence; риск backup=0 по дополнительному диску остаётся.\n- Nextcloud VM150 имеет Proxmox backup и restore-proof evidence.\n- CodeVipe forum-prod имеет локальный backup каждые 6 часов и cloud backup с pve02 каждые 6 часов.\n\nБЕЗОПАСНОСТЬ_КРАТКО\n- Финальная строгая проверка audit-файлов: STRICT_POSSIBLE_SECRET_COUNT=0.\n- Redacted DNS configs сохранены в 28_dns_configs_redacted.txt.\n- Справочник 31 проверен: REFERENCE_ASSIGNMENT_SECRET_HITS=0.\n\n## FINAL_CLOSURE_20260630_CRITICAL_TAILS\n\n- VM160 backup: closed, nightly includes VM160, manual backup ZSTD_OK.\n- VM130 backup: closed, scsi1 backup=1, manual backup ZSTD_OK, exact offhost ZSTD_OK, old local backup removed.\n- AdGuard rewrite sync: closed, STATUS=OK, timer active/enabled.\n- Cloudflare token: live NPMplus token rotated, API verify OK, old exposed token externally confirmed revoked.\n- XenForo SMTP: password rotated, JSON valid, STARTTLS SMTP AUTH OK.\n- Final audit: unredacted secret strict scan OK.\n- Proof index: 85_FINAL_CLOSURE_PROOF.txt and 87_FINAL_CLOSURE_PROOF_WITH_EXTERNAL_CONFIRMATION.txt.\n\n## FINAL_DASHBOARD_RUNTIME_OK_20260630\n\n- Runtime dashboard: STATUS=OK, ITEMS=69, NOT_OK=0.\n- systemd failed units: 0 loaded units listed.\n- SLO/coverage WARN files: moved out of runtime dashboard into slo-coverage-backlog-index.txt.\n- Backlog index: STATUS=OK, BACKLOG_WARN_COUNT=12, RUNTIME_BLOCKER=NO.\n- Final proof: 103_FINAL_ALL_GREEN_DASHBOARD_PROOF.txt.\n## ROUTER_NETCRAZE_ULTRA_NC1812_20260630\n\nИсточник: self-test Netcraze Ultra NC-1812 от 2026-06-30. Секреты Wi-Fi, пароли и токены в справочник не вносятся.\n\n### Паспорт\n- Model: Netcraze Ultra.\n- Device description: Netcraze Ultra (NC-1812).\n- Hostname: Netcraze-9202.\n- Workgroup/domain: WORKGROUP.\n- Timezone: Europe/Moscow.\n- NTP: master.\n- NDNS/caption: ndns-domain.\n- Локальный домен/сертификат в логах: aleisaevn.netcraze.pro.\n- Firmware/NDM из self-test: 5.01.C.0.0-1, built 2026-06-17.\n- sharing-config version: 2.06.1.\n- Components auto-update: disabled.\n- Auto-update channel: draft.\n- Auto-update schedule0: start 05:00, stop 06:00.\n- EasyConfig: disabled.\n- zram: enabled.\n- IPv4 forwarding: enabled.\n- IPv6 forwarding: enabled.\n- conntrack max entries: 32768.\n- TCP established timeout: 1200.\n- TCP fin timeout: 30.\n- TCP keepalive: 120.\n\n### WAN и резервный интернет\n- Main WAN: GigabitEthernet1, renamed ISP, description Ростел.\n- WAN security-level: public.\n- WAN addressing: DHCP.\n- WAN MTU: 1500.\n- WAN global priority: 700.\n- WAN ping-check profile: default.\n- Ping-check hosts from log/profile: ya.ru, vk.com, dzen.ru over TLS/443.\n- WAN DHCP observed in router log: 95.84.154.183/23, gateway 95.84.154.1.\n- ISP DNS observed in router log: 77.37.251.33, 77.37.255.30.\n- Backup/mobile WAN: CdcEthernet0, description SIM.\n- CdcEthernet0 USB device-id: 12d1 14dc.\n- CdcEthernet0 security-level: public.\n- CdcEthernet0 addressing: DHCP.\n- CdcEthernet0 global priority: 350.\n- CdcEthernet0 observed IP: [PRIVATE_IP]/24 via [PRIVATE_IP].\n\n### LAN / VLAN / bridge\n- Home bridge: Bridge0 renamed Home.\n- Home description: Основная.\n- Home security-level: private.\n- Home IP: [PRIVATE_IP]/24.\n- Home includes: GigabitEthernet0/Vlan1, AccessPoint, AccessPoint_5G, XGigabitEthernet0.\n- Proxmox bridge: Bridge1.\n- Proxmox bridge IP: [PRIVATE_IP]/24.\n- Proxmox security-level: protected.\n- Proxmox includes: GigabitEthernet0/Vlan50, XGigabitEthernet0/Vlan50.\n- Port 1: GigabitEthernet0/0, access VLAN 50.\n- Port 2: GigabitEthernet0/1, access VLAN 1 + trunk VLAN 50.\n- Port 3: GigabitEthernet0/2, access VLAN 1 + trunk VLAN 50.\n- Port 4: GigabitEthernet0/3, access VLAN 1 + trunk VLAN 50.\n- Port 5: XGigabitEthernet0/0, XGigabitEthernet0/Vlan50 enabled.\n\n### Wi-Fi\n- SSID: N9202.\n- 2.4 GHz: WifiMaster0, AccessPoint.\n- 2.4 GHz compatibility: BGN+AX+BE.\n- 2.4 GHz channel width: 40-below.\n- 5 GHz: WifiMaster1, AccessPoint_5G.\n- 5 GHz compatibility: AN+AC+AX+BE.\n- 5 GHz channel width: 160.\n- Auto channel rescan: 00:00 interval 1 hour.\n- Encryption: WPA2 + WPA3.\n- WMM: enabled.\n- Beamforming: enabled.\n- TWT: enabled.\n- DL/UL MU-MIMO: enabled.\n- DL/UL OFDMA: enabled.\n- Spatial reuse: enabled.\n- Band steering: disabled.\n- Extra AP interfaces: present but down.\n- MWS WLAN Home: bands 0 and 1, bind Home, SSID N9202, encryption wpa2+3.\n\n### DHCP\n- Main DHCP pool: _WEBADMIN.\n- DHCP range: [PRIVATE_IP]-[PRIVATE_IP].\n- Default router: [PRIVATE_IP].\n- DHCP DNS: [PRIVATE_IP], [PRIVATE_IP].\n- Lease: 25200 seconds.\n- Bound interface: Home.\n- Guest AP pool: _WEBADMIN_GUEST_AP enabled.\n\n### Static DHCP reservations\n- [PRIVATE_IP] -> 00:e0:70:62:98:fb comp01.\n- [PRIVATE_IP] -> 60:3d:61:d4:57:ea Yandex-Ambi-Lamp.\n- [PRIVATE_IP] -> cc:4b:73:58:9b:c0 Yandex.Station.\n- [PRIVATE_IP] -> 38:8a:06:5b:eb:40 Мой смарт.\n- [PRIVATE_IP] -> a0:9d:c1:84:7b:98 Приставка комната.\n- [PRIVATE_IP] -> ac:ba:c0:51:d3:ca Алиса наша комната.\n- [PRIVATE_IP] -> 54:f1:5f:f3:cd:14 Приставка кухня.\n- [PRIVATE_IP] -> ac:ba:c0:50:71:94 Алиса кухня.\n- [PRIVATE_IP] -> 24:4b:fe:d8:8d:2c Коля смарт ROG-Phone3.\n- [PRIVATE_IP] -> 00:2b:70:9a:a3:cf Коля смарт S21-FE.\n- [PRIVATE_IP] -> 00:08:22:34:26:fc Коля SPIDER-10 первый.\n- [PRIVATE_IP] -> 6c:c7:ec:93:be:dc Света смарт.\n- [PRIVATE_IP] -> 00:1e:8f:b0:d5:af Принтер Canon ч/б.\n- [PRIVATE_IP] -> d8:43:ae:68:a3:a1 Коля комп.\n- [PRIVATE_IP] -> 80:1f:12:69:24:e6 Сигнализация.\n- [PRIVATE_IP] -> 00:2b:70:9a:9e:99 Мой смарт S21-FE.\n- [PRIVATE_IP] -> 74:eb:80:2e:aa:4e Света планшет Galaxy Tab S4.\n- [PRIVATE_IP] -> 84:47:09:6a:cc:ca PVE01.\n- [PRIVATE_IP] -> 6c:1f:f7:28:72:4f PVE02.\n- [PRIVATE_IP] -> 08:26:ae:3b:73:61 PVE03.\n- [PRIVATE_IP] -> bc:24:11:30:d4:0f AdGuard 1.\n- [PRIVATE_IP] -> bc:24:11:5b:7e:d8 AdGuard 2.\n- [PRIVATE_IP] -> bc:24:11:f4:c5:d8 WireGuard.\n- [PRIVATE_IP] -> bc:24:11:e1:f3:3c NPMplus / edge-vm.\n- [PRIVATE_IP] -> bc:24:11:e1:9a:25 Nextcloud.\n\n### NAT / port forwarding\n- ISP tcp/80 -> bc:24:11:e1:f3:3c, NPMplus.\n- ISP tcp/443 -> bc:24:11:e1:f3:3c, NPMplus.\n- ISP tcpudp/3478 -> bc:24:11:e1:9a:25, Nextcloud Talk TURN.\n- Home tcp/51820 -> bc:24:11:f4:c5:d8, WireGuard vpn1.\n\n### ACL / firewall\n- isolate-private enabled.\n- _WEBADMIN_Bridge0 permits Home-to-Proxmox access for DNS1/DNS2, AdGuard UI, NPMplus HTTP/HTTPS/UI, Nextcloud AIO/Talk TURN, Proxmox SSH/8006 and ICMP from admin PC.\n- _WEBADMIN_GigabitEthernet1 permits admin PC [PRIVATE_IP] outbound TCP 80/443.\n- Hotspot policy: Home permit, Bridge1 permit; known hosts explicitly permitted.\n\n### Router management\n- HTTP port: 5080.\n- HTTPS port: 5083.\n- HTTP/HTTPS security-level: private.\n- SSH port: 2222.\n- SSH security-level: private.\n- Telnet port configured: 2323, security-level private; router log shows Telnet disabled later on 2026-06-26.\n- Lockout policy for HTTP/Telnet/SSH: 5 15 3.\n- Cloud control2 security-level: private.\n- Admin tags: cli, http, cifs, printers, opt.\n- SFTP denied for admin in log; SSH CLI works.\n\n### Router services\n- service dhcp enabled.\n- service dns-proxy enabled.\n- service igmp-proxy enabled.\n- service http enabled.\n- service cifs enabled.\n- service ssh enabled.\n- service ntp enabled.\n- DNS proxy rebind protection: auto.\n- mDNS reflector: disabled.\n- UPnP LAN: Home.\n- DLNA interface: Home.\n- CIFS share: NTFS-edgtobxf, volume 6F98787822C01361:, automount, permissive.\n- USB storage seen in log: Huawei modem/storage, sdb1 swap around 2G, sdb2 NTFS volume NTFS-edgtobxf.\n\n### Router automation warning\n- Netcraze SSH CLI is not a normal POSIX shell.\n- Logs contain failed commands: while, unset, follow.\n- Automation must use router CLI syntax, not bash syntax.\n## EXTERNAL_SERVICES_UPS_NETBIRD_MAIL_SCRIPTS_20260630\n\n### UPS / NUT\n- UPS hardware discovered on pve01 USB: Cyber Power System CP1500 AVR UPS, USB ID 0764:0501.\n- pve01 role: NUT server + monitor.\n- pve01 enabled NUT units: nut-driver-enumerator, nut-monitor, nut-server, nut-driver.target, nut.target.\n- pve02 role: NUT monitor/client.\n- pve03 role: NUT monitor/client.\n- edge-vm: no UPS/NUT/APCUPSD integration discovered.\n- UPS detailed proof: 107_UPS_DISCOVERY_AND_SETTINGS.txt and 109_UPS_NUT_DETAILED_STATUS_AND_CONFIGS.txt.\n- Secrets from /etc/nut configs must remain redacted in audit output.\n\n### NetBird\n- NetBird service is active on pve01, pve02, pve03 and edge-vm.\n- NetBird version observed: daemon 0.73.2, CLI 0.73.2.\n- pve01: FQDN pve01.netbird.selfhosted, IPv4 100.100.131.41/16.\n- pve02: FQDN pve02.netbird.selfhosted, IPv4 100.100.7.2/16.\n- pve03: FQDN pve03.netbird.selfhosted, IPv4 100.100.34.141/16.\n- edge-vm: FQDN edge-vm.netbird.selfhosted, IPv4 100.100.60.182/16.\n- Interface: wt0, type Kernel.\n- WireGuard port: 51820.\n- Management/Signal: Connected.\n- Relays: 2/2 available.\n- SSH Server through NetBird: Disabled.\n- Observed peers count on listed hosts: 6/9 Connected.\n- Inventory proof: 108_EXTERNAL_SERVICES_NETBIRD_MAIL_SCRIPTS_INVENTORY.txt.\n\n### Forum mail / SMTP\n- forum-prod host: forum-prod.gram1.ru, VM IP [PRIVATE_IP].\n- Local MTA services were not discovered by the first inventory command; forum mail is treated as XenForo external SMTP, not local Postfix/Dovecot.\n- XenForo SMTP rotation proof: 80_xenforo_smtp_rotation_closed.txt.\n- SMTP auth proof: 79_forum_xenforo_smtp_auth_openssl_no_crlf_no_secret.txt.\n- Safe current SMTP inventory: 110_XENFORO_MAIL_EXTERNAL_SMTP_SAFE_INVENTORY.txt.\n- SMTP password must never be printed; store only host, port, SSL mode, auth flag, username length, password length and AUTH OK proof.\n\n### Scripts / automations\n- Full script/unit inventory proof: 108_EXTERNAL_SERVICES_NETBIRD_MAIL_SCRIPTS_INVENTORY.txt.\n- edge-vm owns most application health, dashboard, ingress, backup, NetBox, NPMplus, certificate, Trivy and vulnerability jobs.\n- pve01 owns many backup/offhost/restore/health/security/NetBird VPS/rclone/sops/scrutiny jobs.\n- pve02 owns CodeVipe cloud backup, Vaultwarden isolated restore drill, smartctl textfile and staging/rclone helpers.\n- pve03 owns smartctl textfile, cluster internal IP, NetBird and staging/rclone helpers.\n- All scripts must be treated as managed operational surface; every future edit needs proof file in the active cluster-audit directory.\n## UPS_NUT_DETAILED_CONFIG_20260630\n\n- USB discovery on pve01: Cyber Power System, Inc. CP1500 AVR UPS, USB ID 0764:0501.\n- NUT logical UPS name: cyberpower.\n- NUT-reported device model: BR1000ELCD.\n- NUT manufacturer: CPS.\n- Driver: usbhid-ups.\n- NUT driver version: 2.8.1.\n- CyberPower HID data version: 0.8.\n- NUT USB vendorid/productid: 0764/0501.\n- NUT server node: pve01.\n- NUT server mode: MODE=netserver.\n- NUT server listeners: 127.0.0.1:3493 and [PRIVATE_IP]:3493.\n- NUT clients: pve02 and pve03 in MODE=netclient.\n- pve01 monitor role: MONITOR cyberpower@localhost master.\n- pve02 monitor role: MONITOR cyberpower@[PRIVATE_IP] slave.\n- pve03 monitor role: MONITOR cyberpower@[PRIVATE_IP] slave.\n- Current UPS status at inventory time: OL.\n- Battery charge: 100%.\n- Battery warning threshold: 20%.\n- Battery low threshold: 10%.\n- Runtime estimate: 2544 seconds.\n- Runtime low threshold: 300 seconds.\n- Battery type: PbAcid.\n- Battery voltage: 12.9V nominal 12V.\n- Input voltage: 221.0V nominal 230V.\n- Output voltage: 221.0V.\n- UPS load: 11%.\n- Nominal real power: 600W.\n- Beeper: disabled.\n- Shutdown delay: 20 seconds.\n- Start delay: 30 seconds.\n- Shutdown command on monitored nodes: /sbin/shutdown -h +0.\n- NUT timing: POLLFREQ=5, POLLFREQALERT=5, HOSTSYNC=15, DEADTIME=30, FINALDELAY=5.\n- Powerdown flag: /etc/killpower.\n- Secrets in /etc/nut/upsd.users and upsmon.conf are intentionally redacted in proofs.\n- Proof file: 109_UPS_NUT_DETAILED_STATUS_AND_CONFIGS.txt.\n\n## XENFORO_EXTERNAL_SMTP_CURRENT_20260630\n\n- Host: forum-prod.gram1.ru, VM IP [PRIVATE_IP].\n- Local MTA units: none discovered in inventory output.\n- Mail mode: XenForo external SMTP, not local Postfix/Dovecot.\n- SMTP_HOST=mail.pvepro.ru\n- SMTP_PORT=587\n- SMTP_SSL=false\n- SMTP_AUTH=login\n- USERNAME_LEN=17\n- PASSWORD_LEN=30\n- SECRET_PRINTED=NO\n- SMTP AUTH proof: 79_forum_xenforo_smtp_auth_openssl_no_crlf_no_secret.txt.\n- Rotation closure proof: 80_xenforo_smtp_rotation_closed.txt.\n## TIMERS_AND_SCRIPT_HASHES_REFERENCE_20260630\n\n- Full timers and script SHA256 inventory: 114_TIMERS_AND_SCRIPT_HASHES_INVENTORY.txt.\n- edge-vm timers cover runtime dashboard, Paperless guard, external canary, health metrics, restore drill index, AdGuard rewrite sync, NPMplus cert expiry, NetBox backup/sync, retention, cluster daily status, vulnerability and Trivy scans, ingress hardening and NPMplus admin bind.\n- pve01 timers cover VPN/NetBird health, health metrics, smartctl, disk space, MkDocs refresh, VPS identity audit, storage capacity, quality gate, evidence catalog, backup freshness, docker health, Filebrowser backup/offhost/restore, NPMplus/Kuma backup, NetBird VPS backup/offhost, Authentik/Gitea/Vaultwarden backup, SOPS secret coverage, mail cloud upload/restore, Immich/Memos/Paperless backup/offhost/restore, auto backup, edge-vm vzdump, secret sanity.\n- pve02 timers cover smartctl, CodeVipe cloud backup, dpkg-db backup and Vaultwarden isolated restore drill.\n- pve03 timers cover smartctl and dpkg-db backup.\n- Script hashes were captured for /usr/local/sbin and /usr/local/bin on edge-vm, pve01, pve02 and pve03.\n- Future script edits must create a new proof file and refresh 114_TIMERS_AND_SCRIPT_HASHES_INVENTORY.txt.\n## PROXMOX_CLUSTER_NODE_STORAGE_REFERENCE_20260630\n\n### Cluster\n- Cluster name: homelab.\n- Nodes: 3.\n- Quorum: OK / Quorate Yes.\n- Expected votes: 3.\n- Quorum threshold: 2.\n- Transport: knet.\n- Secure auth: on.\n- Corosync membership: [PRIVATE_IP] pve01, [PRIVATE_IP] pve02, [PRIVATE_IP] pve03.\n- Proxmox VE: pve-manager 9.2.3 on all three nodes.\n- Debian: 13 / trixie on all three nodes.\n- Kernel: 7.0.12-1-pve on all three nodes.\n- Datacenter migration config: secure, network=[PRIVATE_IP]/24.\n- Internal migration/cluster helper IPs: pve01 [PRIVATE_IP], pve02 [PRIVATE_IP], pve03 [PRIVATE_IP].\n\n### Storage policy\n- Storage local: dir /var/lib/vz, content iso/vztmpl/backup/import.\n- Storage local-lvm: lvmthin pool data, content rootdir/images.\n- pve01 local usage at inventory: 32.56%, local-lvm 17.50%.\n- pve02 local usage at inventory: 12.86%, local-lvm 11.65%.\n- pve03 local usage at inventory: 35.79%, local-lvm 64.84%.\n- pve01 staging LV: /mnt/staging, 300G.\n- pve02 staging LV: /mnt/staging, 150G.\n- pve03 staging LV: /mnt/staging, 200G.\n- pve03 local-lvm is the most constrained active VM storage because VM130 has 96G OS disk plus 150G media disk.\n\n### Nightly Proxmox backup job\n- Job: homelab-nightly-all.\n- Schedule: 03:30.\n- Mode: snapshot.\n- Compression: zstd.\n- Storage: local.\n- Enabled: yes.\n- Mail notification: failure.\n- Included VMIDs: 110,111,112,113,130,150,160.\n- Retention: keep-daily=7, keep-weekly=4, keep-monthly=3.\n\n### Node pve01\n- FQDN: pve01.gram1.ru.\n- LAN IP: [PRIVATE_IP]/24.\n- Internal migration IP: [PRIVATE_IP]/24.\n- NetBird IP: 100.100.131.41/16.\n- Bridge: vmbr0 over nic0.\n- CPU: Intel Core i9-12950HX, 24 logical CPUs.\n- RAM: 31Gi.\n- Disk: Lexar SSD NQ7A1 1TB.\n- Main active workloads: CT110 dns1, CT112 unbound1, VM150 nextcloud.\n\n### Node pve02\n- FQDN: pve02.gram1.ru.\n- LAN IP: [PRIVATE_IP]/24.\n- Internal migration IP: [PRIVATE_IP]/24.\n- NetBird IP: 100.100.7.2/16.\n- Bridge: vmbr0 over nic2.\n- CPU: Intel N100, 4 logical CPUs.\n- RAM: 15Gi.\n- Disk: SK800-1TB.\n- Main active workloads: CT111 dns2, CT113 unbound2, VM160 forum-prod.\n\n### Node pve03\n- FQDN: pve03.gram1.ru.\n- LAN IP: [PRIVATE_IP]/24.\n- Internal migration IP: [PRIVATE_IP]/24.\n- NetBird IP: 100.100.34.141/16.\n- Bridge: vmbr0 over nic1.\n- CPU: Intel Core i7-4900MQ, 8 logical CPUs.\n- RAM: 31Gi.\n- Disk: Samsung SSD 870 EVO 500GB.\n- Main active workload: VM130 edge-vm.\n\n### Proof\n- Full raw inventory: 120_PROXMOX_CLUSTER_NODE_STORAGE_NETWORK_INVENTORY.txt.\n## PROXMOX_VM_CT_INVENTORY_REFERENCE_20260630\n\n### CT110 dns1\n- Type: LXC.\n- Node: pve01.\n- Hostname/name: dns1.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:F9:3C:E1.\n- Role: AdGuard Home DNS primary.\n- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameserver: [PRIVATE_IP].\n- Onboot: yes.\n- Startup order: 30.\n- Unprivileged: yes.\n- Backup job: included in homelab-nightly-all.\n\n### CT111 dns2\n- Type: LXC.\n- Node: pve02.\n- Hostname/name: dns2.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:CF:3F:66.\n- Role: AdGuard Home DNS secondary.\n- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameserver: [PRIVATE_IP].\n- Onboot: yes.\n- Startup order: 30.\n- Unprivileged: yes.\n- Backup job: included in homelab-nightly-all.\n\n### CT112 unbound1\n- Type: LXC.\n- Node: pve01.\n- Hostname/name: unbound1.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:22:A6:0D.\n- Role: Unbound recursive resolver primary.\n- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameserver: [PRIVATE_IP].\n- Onboot: yes.\n- Startup order: 20.\n- Unprivileged: yes.\n- Backup job: included in homelab-nightly-all.\n\n### CT113 unbound2\n- Type: LXC.\n- Node: pve02.\n- Hostname/name: unbound2.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:9E:AF:BE.\n- Role: Unbound recursive resolver secondary.\n- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameserver: [PRIVATE_IP].\n- Onboot: yes.\n- Startup order: 20.\n- Unprivileged: yes.\n- Backup job: included in homelab-nightly-all.\n\n### VM130 edge-vm\n- Type: QEMU VM.\n- Node: pve03.\n- Name: edge-vm.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:E1:F3:3C.\n- User: debian.\n- Role: edge application host / reverse proxy / monitoring / backup automation host.\n- Resources: 4 cores, 12GiB RAM.\n- Disks: scsi0 96G OS, scsi1 150G media/data.\n- scsi1 backup flag: backup=1.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameservers: [PRIVATE_IP], [PRIVATE_IP].\n- QEMU guest agent: enabled.\n- Onboot: yes.\n- Startup order: 40.\n- Backup job: included in homelab-nightly-all.\n- Important note: VM130 has exact offhost backup proof after scsi1 backup=1.\n\n### VM150 nextcloud\n- Type: QEMU VM.\n- Node: pve01.\n- Name: nextcloud.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:E1:9A:25.\n- User: debian.\n- Role: Nextcloud AIO.\n- Resources: 4 cores, 8GiB RAM.\n- Disk: scsi0 64G.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameservers: [PRIVATE_IP], [PRIVATE_IP].\n- QEMU guest agent: enabled.\n- Onboot: yes.\n- Startup order: 50.\n- Backup job: included in homelab-nightly-all.\n\n### VM160 forum-prod\n- Type: QEMU VM.\n- Node: pve02.\n- Name: forum-prod.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:B6:45:ED.\n- User: ops.\n- Role: CodeVipe / XenForo production forum.\n- Resources: 2 cores, 4GiB RAM.\n- Disk: scsi0 80G.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameserver in VM config: 1.1.1.1.\n- QEMU guest agent: enabled.\n- Onboot: yes.\n- Startup order: 30.\n- Backup job: included in homelab-nightly-all.\n- Important note: VM160 manual backup proof exists and VM160 is included in nightly job.\n\n### Proof\n- Redacted config inventory: 121_PROXMOX_VM_CT_CONFIG_INVENTORY_REDACTED.txt.\n## DNS_INGRESS_CERTIFICATES_REFERENCE_20260630\n\n### DNS chain\n- Home DHCP gives clients DNS servers: [PRIVATE_IP] and [PRIVATE_IP].\n- dns1: CT110, AdGuard Home, IP [PRIVATE_IP].\n- dns2: CT111, AdGuard Home, IP [PRIVATE_IP].\n- unbound1: CT112, IP [PRIVATE_IP], listens on [PRIVATE_IP]:5335 and 127.0.0.1:5335.\n- unbound2: CT113, IP [PRIVATE_IP], listens on [PRIVATE_IP]:5335 and 127.0.0.1:5335.\n- dns1 upstream: [PRIVATE_IP]:5335.\n- dns2 upstream: [PRIVATE_IP]:5335.\n- AdGuard bootstrap DNS: 1.1.1.1 and 9.9.9.9.\n- AdGuard filtering_enabled=true, rewrites_enabled=true, protection_enabled=true.\n- AdGuard ratelimit=20.\n- Unbound access model: localhost allowed, matching AdGuard IP allowed, rest of [PRIVATE_IP]/24 refused.\n- Unbound do-ip6: no.\n- Unbound private-address includes [PRIVATE_IP]/16, [PRIVATE_IP]/12 and [PRIVATE_IP]/8.\n\n### AdGuard rewrites\n- turn.gram1.ru -> [PRIVATE_IP].\n- git.gram1.ru -> [PRIVATE_IP].\n- dozzle.gram1.ru -> [PRIVATE_IP].\n- paper.gram1.ru -> [PRIVATE_IP].\n- memos.gram1.ru -> [PRIVATE_IP].\n- photos.gram1.ru -> [PRIVATE_IP].\n- auth.gram1.ru -> [PRIVATE_IP].\n- backup.gram1.ru -> [PRIVATE_IP].\n- Rewrite sync health proof: adguard-rewrite-sync STATUS=OK, ORIGIN_COUNT=8, REPLICA_COUNT=8, CHANGED=0.\n- Proof files: 127_ADGUARD_UNBOUND_DNS_SAFE_INVENTORY.txt and 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt.\n\n### Router ingress\n- Public WAN router forwards TCP/80 and TCP/443 to NPMplus on edge-vm, [PRIVATE_IP].\n- Router forwards TCP/UDP 3478 to Nextcloud Talk TURN on [PRIVATE_IP].\n- Router forwards Home TCP/51820 to WireGuard host [PRIVATE_IP].\n- NPMplus admin listener is bound to localhost on edge-vm, 127.0.0.1:81; public disabled legacy host npm.gram1.ru exists but enabled=0.\n\n### NPMplus runtime\n- Host: edge-vm, [PRIVATE_IP].\n- Container: npmplus, image zoeyvid/npmplus:2026-06-17-b1, healthy at inventory time.\n- Socket proxy: lscr.io/linuxserver/socket-proxy:latest, 127.0.0.1:2375.\n- Database: /opt/npmplus/npmplus/database.sqlite.\n- DB integrity: ok.\n- Public listen ports on edge-vm: 0.0.0.0:80 and 0.0.0.0:443 by nginx/NPMplus.\n- NPMplus admin: 127.0.0.1:81.\n- Secret rule: Cloudflare DNS API token exists only inside NPMplus certificate metadata and must never be printed.\n\n### Public NPMplus proxy hosts\n- nc.gram1.ru -> http://[PRIVATE_IP]:11000, cert=17, ssl_forced=1, enabled=1.\n- uptime.gram1.ru -> http://127.0.0.1:3001, cert=35, ssl_forced=1, enabled=1.\n- gotify.gram1.ru -> http://127.0.0.1:8082, cert=35, ssl_forced=1, enabled=1.\n- vault.gram1.ru -> http://127.0.0.1:8083, cert=21, ssl_forced=1, enabled=1.\n- dockge.gram1.ru -> http://127.0.0.1:5001, cert=26, ssl_forced=1, enabled=1.\n- home.gram1.ru -> http://127.0.0.1:3000, cert=27, ssl_forced=1, enabled=1.\n- git.gram1.ru -> http://127.0.0.1:3002, cert=29, ssl_forced=1, enabled=1.\n- dozzle.gram1.ru -> http://127.0.0.1:9999, cert=30, ssl_forced=1, enabled=1.\n- paper.gram1.ru -> http://127.0.0.1:8010, cert=30, ssl_forced=1, enabled=1.\n- memos.gram1.ru -> http://127.0.0.1:5230, cert=30, ssl_forced=1, enabled=1.\n- photos.gram1.ru -> http://127.0.0.1:2283, cert=30, ssl_forced=1, enabled=1.\n- auth.gram1.ru -> http://127.0.0.1:9000, cert=30, ssl_forced=1, enabled=1.\n- backup.gram1.ru -> http://[PRIVATE_IP]:9101, cert=31, ssl_forced=1, enabled=1.\n\n### VPN NPMplus proxy hosts\n- kuma.vpn.gram1.ru -> http://127.0.0.1:3001, cert=32.\n- homepage.vpn.gram1.ru -> http://127.0.0.1:3000, cert=32.\n- gitea.vpn.gram1.ru -> http://127.0.0.1:3002, cert=32.\n- filebrowser.vpn.gram1.ru -> http://127.0.0.1:8085, cert=32.\n- auth.vpn.gram1.ru -> http://127.0.0.1:9000, cert=32.\n- actual-budget.vpn.gram1.ru -> http://127.0.0.1:5006, cert=32.\n- grafana.vpn.gram1.ru -> http://127.0.0.1:3003, cert=32.\n- homebox.vpn.gram1.ru -> http://127.0.0.1:7745, cert=32.\n- mealie.vpn.gram1.ru -> http://127.0.0.1:9001, cert=32.\n- n8n.vpn.gram1.ru -> http://127.0.0.1:5678, cert=32.\n- netbox.vpn.gram1.ru -> http://127.0.0.1:8000, cert=32.\n- node-red.vpn.gram1.ru -> http://127.0.0.1:1880, cert=32.\n- beszel.vpn.gram1.ru -> http://127.0.0.1:8090, cert=32.\n- it-tools.vpn.gram1.ru -> http://127.0.0.1:8084, cert=32.\n- karakeep.vpn.gram1.ru -> http://127.0.0.1:3030, cert=32.\n- linkding.vpn.gram1.ru -> http://127.0.0.1:9091, cert=32.\n- minio.vpn.gram1.ru -> http://127.0.0.1:9003, cert=32.\n- ntfy.vpn.gram1.ru -> http://127.0.0.1:8055, cert=32.\n- searxng.vpn.gram1.ru -> http://127.0.0.1:8888, cert=32.\n- syncthing.vpn.gram1.ru -> http://127.0.0.1:8384, cert=32.\n- alertmanager.vpn.gram1.ru -> http://127.0.0.1:9093, cert=32.\n- prometheus.vpn.gram1.ru -> http://127.0.0.1:9090, cert=32.\n- homeassistant.vpn.gram1.ru -> http://127.0.0.1:8123, cert=32.\n- npmplus.vpn.gram1.ru -> https://[PRIVATE_IP]:81, cert=32.\n- proxmox.vpn.gram1.ru -> https://[PRIVATE_IP]:8006, cert=32.\n- dns1.vpn.gram1.ru -> http://[PRIVATE_IP]:8080, cert=32.\n- dns2.vpn.gram1.ru -> http://[PRIVATE_IP]:8080, cert=32.\n- checks.vpn.gram1.ru -> http://127.0.0.1:8015, cert=32.\n- vikunja.vpn.gram1.ru -> http://127.0.0.1:8016, cert=32.\n- bookstack.vpn.gram1.ru -> http://127.0.0.1:8017, cert=32.\n- pdf.vpn.gram1.ru -> http://127.0.0.1:8018, cert=32.\n- jellyfin.vpn.gram1.ru -> http://127.0.0.1:8019, cert=32.\n- audiobooks.vpn.gram1.ru -> http://127.0.0.1:8020, cert=32.\n- books.vpn.gram1.ru -> http://127.0.0.1:8021, cert=32.\n\n### NPMplus certificates\n- cert=17: nc.gram1.ru, expires 2026-09-12 21:58:35.\n- cert=21: vault.gram1.ru, expires 2026-09-13 13:43:23.\n- cert=26: dockge.gram1.ru, expires 2026-09-13 14:51:59.\n- cert=27: home.gram1.ru, expires 2026-09-13 16:43:44.\n- cert=29: git.gram1.ru, expires 2026-09-13 17:36:55.\n- cert=30: dozzle.gram1.ru, paper.gram1.ru, memos.gram1.ru, photos.gram1.ru, auth.gram1.ru, expires 2026-09-13 23:12:00.\n- cert=31: backup.gram1.ru, expires 2026-09-14 05:04:53.\n- cert=32: *.vpn.gram1.ru and vpn.gram1.ru, expires 2026-09-15 23:18:29.\n- cert=35: uptime.gram1.ru and gotify.gram1.ru, expires 2026-09-19 21:21:59.\n- Deleted/old cert rows observed: cert=1 nc.gram1.ru, cert=18 vault.gram1.ru.\n\n### Proof\n- NPMplus schema/listen inventory: 126_DNS_INGRESS_CERT_NPMPLUS_SAFE_INVENTORY.txt.\n- NPMplus exact proxy/cert rows: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt.\n- AdGuard/Unbound inventory: 127_ADGUARD_UNBOUND_DNS_SAFE_INVENTORY.txt.\n- AdGuard upstream/rewrites and Unbound rules: 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt.\n## EDGE_DOCKER_STACKS_REFERENCE_20260630\n\n### Runtime\n- Host: edge-vm, IP [PRIVATE_IP].\n- Docker Server version: 29.6.0.\n- Docker Compose version: v5.1.4.\n- Primary stack root: /opt/stacks.\n- Additional compose roots: /opt/npmplus-compose, /opt/gotify-compose, /opt/uptime-kuma-compose, /opt/vaultwarden-compose, /opt/dockge-compose.\n- Public ingress terminates through NPMplus on ports 80/443.\n- Most app containers expose only 127.0.0.1 ports and are published through NPMplus.\n- NPMplus uses host networking.\n- socket-proxy exposes Docker API read-oriented surface on 127.0.0.1:2375 and socket-proxy_net.\n- Secret values are not stored in the reference. Only .env/secret file paths are inventoried.\n\n### Compose projects observed\n- actual-budget.\n- audiobookshelf.\n- authentik.\n- beszel.\n- blackbox-exporter.\n- bookstack.\n- cadvisor.\n- calibre-web.\n- crowdsec.\n- diun.\n- dockge-compose.\n- dozzle.\n- filebrowser.\n- gitea.\n- gotify-compose.\n- healthchecks.\n- homeassistant.\n- homebox.\n- homepage.\n- immich.\n- it-tools.\n- jellyfin.\n- karakeep.\n- kopia.\n- linkding.\n- loki-alloy.\n- mealie.\n- memos.\n- minio.\n- n8n.\n- netbox.\n- node-red.\n- npmplus.\n- ntfy.\n- observability-lite.\n- paperless.\n- searxng.\n- socket-proxy.\n- stirling-pdf.\n- syncthing.\n- uptime-kuma-compose.\n- vaultwarden-compose.\n- vikunja.\n\n### Critical app groups\n- Ingress/security: npmplus, socket-proxy, crowdsec.\n- Identity/secrets: authentik, vaultwarden.\n- Monitoring/alerting: uptime-kuma, gotify, healthchecks, observability-lite, loki-alloy, blackbox-exporter, cadvisor, node-exporter, dozzle, diun.\n- Data apps: immich, paperless, memos, gitea, netbox, actual-budget, homebox, mealie, n8n, node-red, vikunja, bookstack.\n- Files/media: nextcloud external VM, immich, filebrowser, syncthing, jellyfin, audiobookshelf, calibre-web, stirling-pdf, minio.\n- Admin/dashboard: homepage, dockge, netbox, dozzle, NPMplus VPN admin.\n\n### Notable local ports\n- Homepage: 127.0.0.1:3000.\n- Uptime Kuma: 127.0.0.1:3001.\n- Gitea: 127.0.0.1:3002.\n- Grafana: 127.0.0.1:3003.\n- Actual Budget: 127.0.0.1:5006.\n- n8n: 127.0.0.1:5678.\n- Paperless: 127.0.0.1:8010.\n- Healthchecks: 127.0.0.1:8015.\n- Vikunja: 127.0.0.1:8016.\n- BookStack: 127.0.0.1:8017.\n- Stirling PDF: 127.0.0.1:8018.\n- Jellyfin: 127.0.0.1:8019.\n- Audiobookshelf: 127.0.0.1:8020.\n- Calibre-Web: 127.0.0.1:8021.\n- ntfy: 127.0.0.1:8055.\n- Gotify: 127.0.0.1:8082.\n- Vaultwarden: 127.0.0.1:8083.\n- IT-Tools: 127.0.0.1:8084.\n- Filebrowser: 127.0.0.1:8085.\n- Beszel: 127.0.0.1:8090.\n- Prometheus: 127.0.0.1:9090.\n- Linkding: 127.0.0.1:9091.\n- Alertmanager: 127.0.0.1:9093.\n- Node exporter: 127.0.0.1:9100.\n- Blackbox exporter: 127.0.0.1:9115.\n- Syncthing UI: 127.0.0.1:8384.\n- Loki: 127.0.0.1:3100.\n- Alloy UI: 127.0.0.1:12345.\n- Home Assistant: host network, 0.0.0.0:8123.\n- NPMplus admin: 127.0.0.1:81.\n- NPMplus public ingress: 0.0.0.0:80 and 0.0.0.0:443.\n\n### Secret/env inventory policy\n- Full values from .env, secrets.yaml, secretkey.txt and token files must never be copied into reference.\n- Reference may list paths only.\n- Full path inventory is in 133_EDGE_DOCKER_STACKS_SAFE_INVENTORY.txt.\n- Normalized runtime map is in 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt.\n\n### Proof\n- Raw Docker/stacks inventory: 133_EDGE_DOCKER_STACKS_SAFE_INVENTORY.txt.\n- Normalized container map: 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt.\n## BACKUP_RESTORE_OFFHOST_CATALOG_REFERENCE_20260630\n\n### Global backup model\n- Primary Proxmox backup job: homelab-nightly-all.\n- Schedule: daily 03:30.\n- Mode: snapshot.\n- Compression: zstd.\n- Storage: local.\n- Retention: keep-daily=7, keep-weekly=4, keep-monthly=3.\n- Included VMIDs: 110,111,112,113,130,150,160.\n- Backup health dashboard: STATUS=OK, ITEMS=69, NOT_OK=0 at 2026-06-30T10:10:06Z.\n- Restore drill index: STATUS=OK, RESTORE_PROOFS=16/16.\n- SLO coverage is tracked separately from runtime dashboard. It had WARN backlog in backup-restore-coverage-slo with APP_TOTAL=43, GREEN=3, YELLOW=21, RED=19, MISSING_BACKUP_SIGNAL=31, MISSING_RESTORE_SIGNAL=28, MISSING_TIMER=31.\n- Runtime OK means operational backup/restore checks are green; it does not mean every P1/P2 application has full backup+restore automation coverage.\n\n### Proxmox vzdump catalog\n- CT110 dns1: included in homelab-nightly-all, local backup on pve01.\n- CT111 dns2: included in homelab-nightly-all, local backup on pve02.\n- CT112 unbound1: included in homelab-nightly-all, local backup on pve01.\n- CT113 unbound2: included in homelab-nightly-all, local backup on pve02.\n- VM130 edge-vm: included in homelab-nightly-all, local backup on pve03.\n- VM150 nextcloud: included in homelab-nightly-all, local backup on pve01.\n- VM160 forum-prod: included in homelab-nightly-all, local backup on pve02.\n- VM130 also has dedicated edge-vm-vzdump backup/offhost/restore health proofs.\n- VM160 has manual backup proof and nightly job inclusion proof.\n- VM130 scsi1 backup flag is enabled after correction: scsi1 backup=1.\n\n### Edge VM / VM130 full-image protection\n- edge-vm-vzdump-backup: STATUS=OK, archive size about 28.2G, SHA256 recorded.\n- edge-vm-vzdump-offhost: STATUS=OK, destination pve02 /mnt/staging/offhost/edge-vm-vzdump-from-pve03.\n- edge-vm-vzdump-restore: STATUS=OK, zstd and vma verification OK.\n- mail-cloud-edge-vm: STATUS=OK, recurring chunked upload, 53 parts, download verification enabled.\n- Retention for edge-vm cloud upload: RETENTION_KEEP=4.\n- Edge restore proof offhost: STATUS=OK, ITEMS=14, OK=14.\n\n### Mail/cloud critical backups\n- mail-cloud-critical: STATUS=OK, recurring P0 critical baseline, remote OK 2/2.\n- Latest observed critical bundle: p0-critical-baseline-20260630T034644Z.tar.gz.\n- Critical bundle size at inventory: 1087208837 bytes.\n- Critical retention: RETENTION_KEEP=14.\n- mail-cloud-restore-drill: STATUS=OK, CRITICAL_OK=2, EDGE_OK=1.\n- vaultwarden isolated restore drill on pve02: STATUS=OK, uses cloud critical baseline plus pve02 offhost component and vaultwarden archive.\n\n### NPMplus / Kuma / ingress config backups\n- npmplus-kuma-config-backup: STATUS=OK.\n- Archive: /mnt/staging/npmplus-kuma-config-backups/snapshots/npmplus-kuma-config-*.tar.gz.\n- NPMplus DB integrity: OK.\n- Kuma DB integrity: OK.\n- NPM proxy count in health proof: 47.\n- Required VPN routes in health proof: 18.\n- Wildcard VPN cert check: NPM_CERT32_WILDCARD=1.\n- npmplus-kuma-config-offhost: STATUS=OK to pve02.\n- npmplus-kuma-config-restore: STATUS=OK with DB integrity checks.\n- npmplus restore proof also exists from app backup quality checks.\n\n### DNS / AdGuard / Unbound protection\n- AdGuard rewrite sync: STATUS=OK, ORIGIN_COUNT=8, REPLICA_COUNT=8, CHANGED=0.\n- dns1/dns2/unbound1/unbound2 are protected by Proxmox vzdump.\n- AdGuard rewrite sync plan stored under /var/lib/homelab-private/adguard-rewrite-sync.\n- Router self-test config is stored as uploaded/generated reference evidence, not yet automated as recurring router backup.\n\n### Auth / secrets / identity apps\n- Vaultwarden backup: STATUS=OK, archive under /mnt/staging/vaultwarden-backups/snapshots.\n- Vaultwarden offhost: STATUS=OK to pve02.\n- Vaultwarden restore: STATUS=OK, DB integrity OK, tables counted.\n- Vaultwarden isolated restore drill: STATUS=OK on pve02.\n- Authentik backup: STATUS=OK, archive under /mnt/staging/authentik-backups/snapshots.\n- Authentik offhost: STATUS=OK to pve02.\n- Authentik restore: STATUS=OK, tables/users/applications/proxy providers checked.\n- Secret values must remain out of all reference files; only archive paths, sizes and SHA256 are recorded.\n\n### Git / documentation / inventory apps\n- Gitea backup: STATUS=OK, archive under /mnt/staging/gitea-backups/snapshots.\n- Gitea offhost: STATUS=OK to pve02.\n- Gitea restore: STATUS=OK, DB integrity OK, tables counted.\n- NetBox backup: STATUS=OK, PostgreSQL dump under /var/lib/homelab-private/pre-update-backups/netbox-*.\n- NetBox offhost: STATUS=OK to pve02.\n- NetBox restore dry-run: STATUS=OK, restore container checked.\n- Latest NetBox health proof observed: devices=7, sites=1, prefixes=3, IP addresses=12, virtual machines=49.\n\n### Document / notes / personal data apps\n- Paperless backup: STATUS=OK, latest snapshot under /mnt/staging/paperless-backups/snapshots.\n- Paperless offhost: STATUS=OK to pve02.\n- Paperless restore dry-run: STATUS=OK, pg_restore list and restore tables checked.\n- Memos backup: STATUS=OK, latest snapshot under /mnt/staging/memos-backups/snapshots.\n- Memos offhost: STATUS=OK to pve02.\n- Memos restore proof exists from app restore quality checks.\n- Actual Budget backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof.\n- Linkding backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof.\n- Karakeep backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof.\n- Mealie backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof.\n- n8n backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof.\n\n### Files / media / sync apps\n- Immich media backup: STATUS=OK, latest snapshot under /mnt/staging/immich-media-backups/snapshots.\n- Immich media offhost: STATUS=OK to pve02.\n- Immich media restore: STATUS=OK, local/offhost manifest match.\n- Immich full consistency backup: STATUS=OK, includes PostgreSQL dump and media archive, offhost path present, restore dry-run OK.\n- Nextcloud restore proof: STATUS=OK, OCC OK, DB_DUMP OK, DATA_MANIFEST OK, restore to production NO.\n- Nextcloud restore proof is copied offhost.\n- Filebrowser backup: STATUS=OK.\n- Filebrowser offhost: STATUS=OK.\n- Filebrowser restore validation: STATUS=OK.\n- Jellyfin restore: STATUS=OK from app backup quality checks.\n- Audiobookshelf restore: STATUS=OK from app backup quality checks.\n- Calibre-Web restore: STATUS=OK from app backup quality checks.\n- Syncthing is inventoried in Docker/stacks; dedicated backup signal not confirmed in this catalog.\n\n### Home/admin/utility apps\n- Home Assistant backup: STATUS=OK, archive includes configuration.yaml, storage dir and DB file, offhost SHA256/listing OK, restore not attempted in observed proof.\n- Healthchecks restore: STATUS=OK, SQLite integrity OK and temp restore verify OK.\n- BookStack restore: STATUS=OK, DB dump OK.\n- Vikunja restore: STATUS=OK, SQLite integrity OK and temp restore verify OK.\n- Kopia restore: STATUS=OK, synthetic snapshot and restore compare OK.\n- Observability config backup: STATUS=OK, includes prometheus.yml, alertmanager.yml and health rules, offhost SHA256/listing OK.\n- Ntfy is covered as part of observability/alerting stack but dedicated service backup signal was not confirmed in this catalog.\n- MinIO is included in app-stacks-batch backup from 20260617; current dedicated timer was not confirmed.\n- Homebox, Node-RED, MinIO, Mealie, Linkding, n8n and Actual Budget were part of app-stacks batch backup proof from 20260617.\n\n### External service backups\n- NetBird VPS backup: STATUS=OK, snapshot under /mnt/staging/netbird-vps-backups/snapshots.\n- NetBird VPS offhost: STATUS=OK to pve02.\n- NetBird VPS restore validation: STATUS=OK, archive SHA256 recorded.\n- CodeVipe/forum-prod: VM160 Proxmox backup is present; pve02 also has codevipe-cloud-backup.timer.\n- XenForo SMTP rotation has DB backup proof from earlier rotation work; production forum application-level backup catalog should be expanded separately with CodeVipe-specific DB/files/cron/public cutover details.\n\n### Retention and cleanup\n- Retention dry-run: STATUS=OK, production changed NO, destructive actions NO.\n- Disk retention policy: STATUS=OK, root used pct observed 70 on edge-vm, removed dirs 0, Docker volume prune NO.\n- App backup retention dry-run timer exists on edge-vm.\n- homelab-backup-freshness timer exists on pve01.\n- rollback-image-retention health file exists but had permission issue during one raw read; it must be read with sudo in future collection commands.\n\n### Known coverage gaps / backlog\n- backup-restore-coverage-slo remains a coverage backlog, not a runtime blocker.\n- Actual Budget latest proof says RESTORE_ATTEMPTED=NO.\n- Home Assistant latest proof says RESTORE_ATTEMPTED=NO.\n- Linkding latest proof says RESTORE_ATTEMPTED=NO.\n- Karakeep latest proof says RESTORE_ATTEMPTED=NO.\n- Mealie latest proof says RESTORE_ATTEMPTED=NO.\n- n8n latest proof says RESTORE_ATTEMPTED=NO.\n- Observability config latest proof says RESTORE_ATTEMPTED=NO.\n- Router config backup is currently reference/self-test based, not confirmed as recurring automated backup.\n- Syncthing, ntfy, MinIO and several P2 services need either dedicated backup/restore proof or explicit classification as disposable/rebuildable.\n\n### Proof\n- Raw backup/offhost/restore inventory: 138_BACKUP_RESTORE_OFFHOST_CATALOG_RAW_INVENTORY.txt.\n- Backup timer summary draft: 139_BACKUP_CATALOG_SERVICE_SUMMARY_DRAFT.txt.\n- Normalized backup health index: 140_BACKUP_HEALTH_FILES_NORMALIZED_INDEX.txt.\n## MONITORING_ALERTING_HEALTH_REFERENCE_20260630\n\n### Monitoring stack\n- Primary monitoring host: edge-vm, [PRIVATE_IP].\n- Prometheus container: prometheus, local port 127.0.0.1:9090.\n- Grafana container: grafana, local port 127.0.0.1:3003, VPN route grafana.vpn.gram1.ru.\n- Alertmanager container: alertmanager, local port 127.0.0.1:9093.\n- Loki container: loki, local port 127.0.0.1:3100.\n- Alloy container: alloy, local port 127.0.0.1:12345.\n- Uptime Kuma container: uptime-kuma, local port 127.0.0.1:3001, public route uptime.gram1.ru and VPN route kuma.vpn.gram1.ru.\n- Gotify container: gotify, local port 127.0.0.1:8082, public route gotify.gram1.ru.\n- ntfy container: ntfy, local port 127.0.0.1:8055, VPN route ntfy.vpn.gram1.ru.\n- Healthchecks container: healthchecks, local port 127.0.0.1:8015, VPN route checks.vpn.gram1.ru.\n- Blackbox exporter: blackbox-exporter, local port 127.0.0.1:9115.\n- cAdvisor: cadvisor, local port 127.0.0.1:8081.\n- Node exporter on edge-vm: node-exporter, local port 127.0.0.1:9100.\n- Beszel: beszel, local port 127.0.0.1:8090.\n- Dozzle: dozzle, local port 127.0.0.1:9999.\n\n### PVE monitoring endpoints\n- pve01: prometheus-node-exporter active on :9100, homelab-health-http active on :9101, Proxmox pveproxy on :8006.\n- pve02: prometheus-node-exporter active on :9100, homelab-health-http active on :9101, Proxmox pveproxy on :8006.\n- pve03: prometheus-node-exporter active on :9100, Proxmox pveproxy on :8006.\n- pve03 has node-exporter/SMART timers but no homelab-health-http :9101 endpoint in observed output.\n- PVE smartctl textfile timers exist on pve01/pve02/pve03.\n- pve01 also runs private VPN host health, NetBird peer health, disk space health, MkDocs refresh, evidence catalog and quality gate timers.\n\n### Prometheus config\n- Prometheus scrape interval: 30s.\n- Prometheus evaluation interval: 30s.\n- Rule files path: /etc/prometheus/rules/*.yml.\n- Alertmanager target: alertmanager:9093.\n- Blackbox HTTP job targets:\n - https://nc.gram1.ru\n - https://git.gram1.ru\n - https://auth.gram1.ru\n - https://paper.gram1.ru\n - https://backup.gram1.ru\n- Edge node exporter scrape target: node-exporter:9100.\n- PVE node exporter targets: [PRIVATE_IP]:9100, [PRIVATE_IP]:9100, [PRIVATE_IP]:9100.\n- Proxmox exporter job targets pve01/pve02/pve03 through exporter endpoint [PRIVATE_IP]:9221.\n- cAdvisor scrape target: cadvisor:8080.\n\n### Alertmanager / notification routing\n- Alertmanager route receiver: ntfy.\n- Alertmanager webhook target: http://ntfy/homelab-alerts.\n- Alert group_by: alertname, instance, severity.\n- group_wait: 10s.\n- group_interval: 5m.\n- repeat_interval: 4h.\n- Alert routing proof: alert-routing.txt STATUS=OK, RULES_ADDED=6, PROMETHEUS_RELOAD=OK, TEST_NOTIFICATION_SENT=YES, NTFY_MESSAGES_PUBLISHED_DELTA=1.\n- alert-routing-health.txt is the standardized status alias and is STATUS=OK.\n\n### Core Prometheus alert rules\n- HomelabPrometheusTargetDown: up == 0 for 5m, severity warning.\n- HomelabSmartDiskHealthFailed: homelab_smartctl_health_passed == 0 for 10m, severity critical.\n- HomelabRootFilesystemLowSpace: root filesystem available below 15% for 15m, severity warning.\n- HomelabPrometheusConfigReloadFailed: prometheus_config_last_reload_successful == 0 for 5m, severity critical.\n- HomelabP0HealthStatusNotOk: homelab_health_status tier P0 equals 0 for 15m.\n- HomelabP0BackupRestoreHealthStale: P0 backup/offhost/restore health older than 36h.\n- HomelabP0WeeklyEdgeVmVzdumpHealthStale: weekly edge-vm vzdump health older than 8d.\n- HomelabP1BackupHealthStale: P1 backup health older than 7d.\n- HomelabMailCloudRestoreDrillStale: mail-cloud restore drill older than 8d.\n- HomelabMkdocsRuntimePublishStale: MkDocs runtime publish older than 30d.\n- HomelabHealthMetricsCollectorMissing: homelab health metrics absent from Prometheus.\n- HomelabExternalCanaryHealthNotOkOrStale: external canary not OK or older than 30m.\n- HomelabBlackboxProbeFailed / Missing: blackbox probe failed or absent.\n- HomelabVulnScanHealthNotOkOrStale: vulnerability scan failed or older than 8d.\n- HomelabNpmplusCertExpiryHealthNotOkOrStale: NPMplus cert expiry health failed or older than 48h.\n- HomelabBlackboxTlsCertificateExpiringSoon: TLS certificate expiry within 14 days.\n\n### Loki / Alloy log pipeline\n- Loki version observed: grafana/loki:3.7.2.\n- Alloy version observed: grafana/alloy:v1.17.0.\n- Loki auth_enabled: false.\n- Loki storage: local filesystem under /loki.\n- Loki schema: tsdb v13.\n- Loki retention_period: 14d.\n- Alloy discovers Docker through socket-proxy at tcp://socket-proxy:2375.\n- Alloy relabels container, compose_project, compose_service and host=edge-vm.\n- Alloy forwards Docker logs to http://loki:3100/loki/api/v1/push.\n- Loki readiness observed as \"ready\".\n\n### Runtime dashboard semantics\n- backup-restore-dashboard.txt is an authoritative runtime dashboard file.\n- Current observed runtime dashboard: STATUS=OK, ITEMS=69, NOT_OK=0.\n- backup-restore-dashboard.json confirmed not_ok=[].\n- restore-drill-index.txt observed: STATUS=OK, RESTORE_PROOFS=16/16, MISSING empty.\n- cluster-daily-status.txt observed: STATUS=OK, PROBLEM_COUNT=0.\n- external-canary.txt observed: STATUS=OK, BAD=0.\n- external canary checks include nc.gram1.ru, git.gram1.ru, auth.gram1.ru, backup.gram1.ru.\n- Alertmanager local API is reachable.\n- Gotify local /health returns green.\n- ntfy local /v1/health returns healthy true.\n- Uptime Kuma local endpoint returns HTTP 302 to /dashboard, which is expected for UI auth/redirect.\n- Healthchecks local status endpoint returns HTTP 301, which is treated as reachable.\n\n### Certificate / vulnerability health\n- npmplus-cert-expiry-health.txt is the standardized health alias for certificate expiry.\n- npmplus-certificate-expiry.txt is the detailed cert expiry file.\n- Detailed cert expiry file observed cert_count=9 and min_cert_days_left=75 with warn threshold 30.\n- cluster-daily-status local extra check reports npmplus-cert-expiry status OK, problems=0, min_days=75.\n- trivy-scan.txt observed: STATUS=OK, IMAGE_TOTAL=58, IMAGE_SCAN_OK=58, IMAGE_SCAN_ERROR=0, CRITICAL_TOTAL=112, HIGH_TOTAL=1650.\n- vuln-scan.txt observed: STATUS=OK, MODE=full, IMAGES=58, TRIVY_OK=49, CRITICAL_MATCHES=203.\n\n### SLO backlog semantics\n- Runtime dashboard OK does not mean SLO backlog is closed.\n- slo-coverage-backlog-index.txt observed: STATUS=OK, BACKLOG_WARN_COUNT=12, RUNTIME_BLOCKER=NO.\n- backup-restore-coverage-slo.txt observed: STATUS=WARN, APP_TOTAL=43, GREEN_COUNT=3, YELLOW_COUNT=21, RED_COUNT=19, MISSING_BACKUP_SIGNAL=31, MISSING_RESTORE_SIGNAL=28, MISSING_TIMER=31, MISSING_EXTERNAL_VALIDATION=43.\n- Other observed WARN backlog files include container-image-lifecycle-slo, cron-job-monitoring-slo, direct-heartbeat-pilot-readiness, healthchecks-heartbeat-coverage, healthchecks-job-coverage, healthchecks-job-monitoring-slo, homepage-service-catalog-slo, ingress-dns-route-slo, observability-health-surface-slo and security-vulnerability-slo.\n- SLO WARN files are tracked separately and are not runtime blockers unless explicitly promoted to P0 runtime checks.\n\n### Important file locations\n- Main health dir on edge-vm: /var/lib/homelab-health.\n- Prometheus config: /opt/stacks/observability-lite/prometheus/prometheus.yml.\n- Prometheus rules: /opt/stacks/observability-lite/prometheus/rules/.\n- Alertmanager config: /opt/stacks/observability-lite/alertmanager/alertmanager.yml.\n- Loki config: /opt/stacks/loki-alloy/loki-config.yaml.\n- Alloy config: /opt/stacks/loki-alloy/config.alloy.\n- Blackbox exporter config: /opt/stacks/blackbox-exporter/blackbox.yml.\n- Uptime Kuma DB: /opt/uptime-kuma-compose/data/kuma.db.\n- Gotify DB: /opt/gotify-compose/data/gotify.db.\n- ntfy DB/cache: /opt/stacks/ntfy/data/user.db and /opt/stacks/ntfy/cache/cache.db.\n- Healthchecks settings/data: /opt/stacks/healthchecks/local_settings.py and /var/lib/healthchecks.\n\n### Known caveats\n- A previous compact check looked for npmplus-cert-expiry.txt; use npmplus-certificate-expiry.txt for detailed cert expiry and npmplus-cert-expiry-health.txt for standardized health.\n- health-metrics.txt is not required to exist as a health file; health metrics may be emitted via node-exporter textfile collector.\n- Prometheus targets must be checked with explicit HTTP code and parsed JSON output, not a silent curl pipeline.\n\n### Proof\n- Raw monitoring inventory: 143_MONITORING_ALERTING_HEALTH_RAW_INVENTORY.txt.\n- Runtime/SLO/alerting status: 144_MONITORING_RUNTIME_SLO_ALERTING_STATUS.txt.\n- Kuma/Gotify/ntfy/Healthchecks status: 145_KUMA_GOTIFY_NTFY_HEALTHCHECKS_SAFE_STATUS.txt.\n- Corrected monitoring compact status: 146_MONITORING_COMPACT_STATUS_CORRECTED.txt.\n\n## PROMETHEUS_STATUS_CORRECTION_20260630\n\n- Initial monitoring compact check after reference creation reported Prometheus API HTTP_CODE=000 on 127.0.0.1:9090.\n- This was recorded as error-register item 30 and checked with proof files 149_PROMETHEUS_DIAG_AFTER_MONITORING_REFERENCE.txt, 150_PROMETHEUS_CONDITIONAL_RECOVERY.txt and 151_PROMETHEUS_REFERENCE_CORRECTION_PROOF.txt.\n- The Monitoring/Alerting section must be interpreted together with the latest Prometheus correction proof, not only with the earlier secret-scan proof.\n\n## COMMAND_PREFLIGHT_RULE_20260630\n\n- Strict operator rule: before every command, check both the error register and this reference file.\n- Required visible markers before main action: ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK.\n- If either check fails, do not run the main action.\n- Do not use exit 1 in interactive SSH sessions.\n- Proof: 152_COMMAND_PREFLIGHT_RULE_RECORDED.txt.\n\n## PROMETHEUS_TARGETS_SETTLED_20260630\n\n- Prometheus was absent after the initial monitoring reference proof and was restarted in 150_PROMETHEUS_CONDITIONAL_RECOVERY.txt.\n- Prometheus readiness proof exists in 151_PROMETHEUS_REFERENCE_CORRECTION_PROOF.txt.\n- Target health was rechecked with an additional settled scrape proof in 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt.\n- Interpret Prometheus status from the latest settled proof, not from the earlier immediate-after-start unknown target state.\n## PROMETHEUS_TARGETS_SETTLED_PROOF_CORRECTION_20260630\n\n- 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt is invalid and must not be used as evidence.\n- Reason: nested Python inside SSH lost quoting and produced SyntaxError.\n- Error-register item 32 records this mistake.\n- Valid replacement proof: 154_PROMETHEUS_TARGETS_SETTLED_PROOF_NO_PYTHON.txt.\n- Prometheus status must be interpreted from 154 and later proofs, not from 153.\n\n## MONITORING_PROMETHEUS_FINAL_CLOSED_20260630\n\n- Prometheus was initially absent after monitoring reference creation.\n- It was restarted and then verified after scrape settling.\n- Valid settled proof: 154_PROMETHEUS_TARGETS_SETTLED_PROOF_NO_PYTHON.txt.\n- Final observed state: HTTP_CODE=200, PROM_TARGETS_TOTAL=14, PROM_UP_COUNT=14, PROM_NON_UP_COUNT=0.\n- Correction/secret scan proof: 156_PROMETHEUS_CORRECTION_PROOF_AND_SECRET_SCAN.txt.\n- Invalid proof 153 must not be used.\n## SECURITY_ACCESS_SECRETS_OPERATING_MODEL_20260630\n\n### Command safety\n- Before every command, check /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md and current 31_HOMELAB_REFERENCE.md.\n- Print ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK before the main action.\n- Do not run the main action if either check fails.\n- Do not use exit 1 in interactive SSH sessions.\n- Avoid long nested SSH/Python/PHP/SQL quoting chains.\n- Do not print secrets.\n\n### Access model\n- Primary operator entrypoint: root@pve01 / [PRIVATE_IP].\n- pve02 and pve03 are reached as root from pve01.\n- edge-vm is reached as debian@[PRIVATE_IP] with sudo.\n- forum-prod is reached through pve02 using [SENSITIVE_PATH] to root@[PRIVATE_IP].\n- PVE users observed: root@pam and prometheus@pve.\n- prometheus@pve has PVEAuditor on / for Proxmox exporter access.\n\n### SSH and permissions\n- pve01 root keys observed: id_rsa, id_ed25519 and public keys.\n- pve02 root keys observed: id_rsa and forum-prod-ci-key.\n- pve03 root key observed: id_rsa.\n- edge-vm root key observed: id_ed25519; debian authorized_keys observed.\n- Security finding: pve01/pve02/pve03 root authorized_keys files were observed as mode 777.\n- Correction proof: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt.\n- Private key material must never be copied into the reference.\n\n### Secret storage\n- Primary private storage root: /var/lib/homelab-private.\n- Edge private secrets root: /var/lib/homelab-private/secrets.\n- NPMplus DB: /opt/npmplus/npmplus/database.sqlite, root-only mode observed.\n- Gotify DB: /opt/gotify-compose/data/gotify.db.\n- Uptime Kuma DB: /opt/uptime-kuma-compose/data/kuma.db.\n- Vaultwarden DB: /opt/vaultwarden-compose/data/db.sqlite3.\n- Rclone configs exist under root config paths and must not be printed.\n- Env/secret inventory is path-only: owner, mode, size and path only.\n\n### Network exposure\n- PVE nodes expose SSH :22, Proxmox :8006 and node-exporter :9100.\n- pve01/pve02 expose homelab-health-http :9101.\n- pve01 exposes NUT :3493 on localhost and [PRIVATE_IP].\n- edge-vm exposes public :80/:443 through NPMplus.\n- edge-vm NPMplus admin :81 and socket-proxy :2375 are bound to 127.0.0.1.\n- edge-vm registry cache :5000 is bound to [PRIVATE_IP].\n- edge-vm Home Assistant :8123 is intentionally LAN-exposed.\n- Edge ingress hardening proof reports STATUS=OK.\n\n### Rotation and scan proofs\n- Cloudflare token rotation completed without printing token values.\n- Old Cloudflare token revocation was externally confirmed.\n- XenForo SMTP password rotation completed without printing password values.\n- XenForo SMTP auth was verified with STARTTLS-safe method.\n- Current reference strict scan shows zero strict secret hits.\n- Selected generated reference blocks show zero strict secret hits.\n\n### Proof\n- Raw security/access inventory: 158_SECURITY_ACCESS_SECRETS_RAW_INVENTORY.txt.\n- Secret scan and rotation proof index: 159_SECURITY_SECRET_SCAN_AND_ROTATION_PROOF_INDEX.txt.\n- SSH permission correction proof: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt.\n## SECURITY_SSH_PERMISSION_CORRECTION_20260630\n\n- Initial proof 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt is not sufficient as closure evidence.\n- Reason: stat without -L still showed 777 after chmod, which may be symlink mode rather than target file mode.\n- Error-register item 35 records this proof-quality issue.\n- Authoritative replacement proof: 163_PVE_ROOT_AUTHORIZED_KEYS_SYMLINK_AWARE_FIX.txt.\n- Valid closure condition: target permissions checked with stat -L are 600 for authorized_keys files and [SENSITIVE_PATH] is 700.\n- Final observed status in replacement proof: STATUS=SSH_AUTHORIZED_KEYS_TARGET_PERMISSIONS_OK.\n## STORAGE_DISK_SMART_CAPACITY_REFERENCE_20260630\n\n### Proxmox storage model\n- Cluster storage is defined in /etc/pve/storage.cfg.\n- local storage: dir /var/lib/vz, content iso,vztmpl,backup,import.\n- local-lvm storage: lvmthin data, content rootdir,images.\n- Nightly vzdump job writes to local storage.\n- VM/CT images are primarily stored on local-lvm.\n- Staging/offhost paths are under /mnt/staging where present.\n\n### Node disks and capacity\n- pve01: primary disk previously inventoried as Lexar SSD NQ7A1 1TB.\n- pve02: primary disk previously inventoried as SK800-1TB.\n- pve03: primary disk previously inventoried as Samsung 870 EVO 500GB.\n- pve01 storage usage was previously observed around local 32.56% and local-lvm 17.50%.\n- pve02 storage usage was previously observed around local 12.86% and local-lvm 11.65%.\n- pve03 storage usage was previously observed around local 35.79% and local-lvm 64.84%.\n- Raw current df/lsblk/LVM/SMART evidence is in 166_STORAGE_DISK_SMART_CAPACITY_RAW_INVENTORY.txt.\n\n### Edge VM storage model\n- edge-vm runs Docker application stacks.\n- Docker app data lives mainly under /opt/stacks, /opt/npmplus, /opt/gotify-compose, /opt/uptime-kuma-compose, /opt/vaultwarden-compose and /var/lib application paths.\n- Immich media is mounted separately at /mnt/immich-media in the container mapping.\n- Docker volume and image usage is captured in docker system df output.\n- Disk retention health is tracked by disk-retention-policy and disk-retention-cleanup files.\n\n### SMART and health monitoring\n- PVE nodes run smartctl textfile timers.\n- pve01/pve02/pve03 expose node-exporter :9100 for Prometheus.\n- pve01 and pve02 expose homelab-health-http :9101.\n- SMART/disk/capacity health evidence is collected from /var/lib/homelab-health where present.\n- Prometheus alert rule HomelabSmartDiskHealthFailed watches homelab_smartctl_health_passed == 0.\n- Prometheus alert rule HomelabRootFilesystemLowSpace watches root filesystem available below 15%.\n- pve03 did not have /var/lib/homelab-health in previous backup-health collection, but it has node-exporter and smartctl timers.\n\n### Retention and cleanup\n- Edge disk retention policy previously observed STATUS=OK.\n- Edge disk retention policy observed ROOT_USED_PCT=70, REMOVED_DIRS=0, DOCKER_VOLUME_PRUNE=NO.\n- Retention dry-run policy is designed to avoid destructive production changes.\n- Broad Docker prune is not used as a default cleanup mechanism.\n- Cleanup and retention actions must have explicit proof files.\n\n### Operational rules\n- Before deleting or pruning storage, create or identify rollback/backup proof.\n- Do not delete vzdump/offhost/cloud backup artifacts unless retention policy and proof explicitly allow it.\n- Do not infer offhost success from a failed rsync.\n- For backup artifacts, verify exact file, size, checksum or zstd/vma integrity where applicable.\n- For SMART, use health output plus selected attributes, not full raw dumps with unnecessary noise.\n\n### Proof\n- Raw storage/disk/SMART/capacity inventory: 166_STORAGE_DISK_SMART_CAPACITY_RAW_INVENTORY.txt.\n- Earlier Proxmox node/storage inventory: 120_PROXMOX_CLUSTER_NODE_STORAGE_NETWORK_INVENTORY.txt.\n- Backup/restore catalog: 141_REFERENCE_BACKUP_RESTORE_OFFHOST_CATALOG_BLOCK.txt.\n- Monitoring rules and SMART alert model: 147_REFERENCE_MONITORING_ALERTING_HEALTH_BLOCK.txt.\n## STORAGE_CAPACITY_CORRECTION_20260630\n\n- Storage block integrity review proof: 169_STORAGE_BLOCK_INTEGRITY_AND_PVE03_CAPACITY_REVIEW.txt.\n- 167_REFERENCE_STORAGE_DISK_SMART_CAPACITY_BLOCK.txt integrity status: OK.\n- pve03 /mnt/staging current observed use percent: 77.\n- pve03 staging is close to the existing WARN threshold of 80% if observed use is 75% or higher.\n- pve01 disk-space.txt coverage visibly listed pve01_staging, pve02_staging and edge_immich_media; pve03_staging coverage must be added or explicitly documented elsewhere.\n- Storage layer is operationally OK only if SMART remains PASSED and pve03 staging is tracked before it crosses WARN/CRIT thresholds.\n- This is a capacity coverage note, not a secret or runtime outage.\n## PVE03_STAGING_CAPACITY_MONITOR_20260630\n\n- pve03 /mnt/staging was observed at 77% usage.\n- Existing pve01 disk-space.txt did not visibly include pve03_staging coverage.\n- A dedicated pve03 staging capacity health check was added on pve01.\n- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt.\n- Timer: homelab-pve03-staging-capacity-health.timer.\n- Service: homelab-pve03-staging-capacity-health.service.\n- Current observed status: OK.\n- WARN threshold: 80%.\n- CRIT threshold: 90%.\n- Proof: 173_PVE03_STAGING_CAPACITY_HEALTH_INSTALLED.txt.\n\n## STORAGE_LAYER_FINAL_CLOSED_WITH_PVE03_MONITOR_20260630\n\n- Storage/disk/SMART/capacity layer is closed with an explicit pve03 staging capacity monitor.\n- pve03 /mnt/staging was near WARN threshold at 77%, so a dedicated health file and timer were added.\n- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt.\n- Timer: homelab-pve03-staging-capacity-health.timer.\n- Final closure proof: 176_STORAGE_FINAL_CLOSURE_WITH_PVE03_MONITOR.txt.\n## SERVICE_DEPENDENCY_MAP_20260630\n\n### Ingress and DNS chain\n- Home DHCP DNS: [PRIVATE_IP] and [PRIVATE_IP].\n- dns1: CT110 / [PRIVATE_IP] / AdGuard Home.\n- dns2: CT111 / [PRIVATE_IP] / AdGuard Home.\n- unbound1: CT112 / [PRIVATE_IP]:5335.\n- unbound2: CT113 / [PRIVATE_IP]:5335.\n- dns1 upstream: [PRIVATE_IP]:5335.\n- dns2 upstream: [PRIVATE_IP]:5335.\n- Public ingress HTTP/HTTPS: router forwards TCP 80/443 to edge-vm [PRIVATE_IP].\n- Reverse proxy: NPMplus on edge-vm, container npmplus, host networking, admin bound to 127.0.0.1:81.\n- Edge runtime host: VM130 edge-vm, [PRIVATE_IP], Docker Compose projects count 43.\n- NPMplus proxy routes count: 47.\n- NPMplus certificates and proxy route source proof: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt.\n\n### Core public routes\n| Route | Upstream | Runtime owner | Backup/health evidence |\n|---|---|---|---|\n| nc.gram1.ru | http://[PRIVATE_IP]:11000 | VM150 Nextcloud AIO | VM150 vzdump, nextcloud restore proof, external canary |\n| uptime.gram1.ru | http://127.0.0.1:3001 | edge-vm / uptime-kuma | npmplus-kuma backup/restore, Kuma health |\n| gotify.gram1.ru | http://127.0.0.1:8082 | edge-vm / gotify | gotify health, alert-routing proof |\n| vault.gram1.ru | http://127.0.0.1:8083 | edge-vm / vaultwarden | vaultwarden backup/offhost/restore |\n| dockge.gram1.ru | http://127.0.0.1:5001 | edge-vm / dockge | stack inventory |\n| home.gram1.ru | http://127.0.0.1:3000 | edge-vm / homepage | homepage container, dashboard route |\n| git.gram1.ru | http://127.0.0.1:3002 | edge-vm / gitea | gitea backup/offhost/restore |\n| dozzle.gram1.ru | http://127.0.0.1:9999 | edge-vm / dozzle | docker stack inventory |\n| paper.gram1.ru | http://127.0.0.1:8010 | edge-vm / paperless | paperless backup/offhost/restore |\n| memos.gram1.ru | http://127.0.0.1:5230 | edge-vm / memos | memos backup/offhost/restore |\n| photos.gram1.ru | http://127.0.0.1:2283 | edge-vm / immich | immich media/full consistency proof |\n| auth.gram1.ru | http://127.0.0.1:9000 | edge-vm / authentik | authentik backup/offhost/restore |\n| backup.gram1.ru | http://[PRIVATE_IP]:9101 | pve01 / homelab-health-http | backup dashboard and health proofs |\n\n### VPN routes on wildcard certificate\n| Route | Upstream | Runtime owner |\n|---|---|---|\n| kuma.vpn.gram1.ru | http://127.0.0.1:3001 | uptime-kuma |\n| homepage.vpn.gram1.ru | http://127.0.0.1:3000 | homepage |\n| gitea.vpn.gram1.ru | http://127.0.0.1:3002 | gitea |\n| filebrowser.vpn.gram1.ru | http://127.0.0.1:8085 | filebrowser |\n| auth.vpn.gram1.ru | http://127.0.0.1:9000 | authentik |\n| actual-budget.vpn.gram1.ru | http://127.0.0.1:5006 | actual-budget |\n| grafana.vpn.gram1.ru | http://127.0.0.1:3003 | grafana |\n| homebox.vpn.gram1.ru | http://127.0.0.1:7745 | homebox |\n| mealie.vpn.gram1.ru | http://127.0.0.1:9001 | mealie |\n| n8n.vpn.gram1.ru | http://127.0.0.1:5678 | n8n |\n| netbox.vpn.gram1.ru | http://127.0.0.1:8000 | netbox |\n| node-red.vpn.gram1.ru | http://127.0.0.1:1880 | node-red |\n| beszel.vpn.gram1.ru | http://127.0.0.1:8090 | beszel |\n| it-tools.vpn.gram1.ru | http://127.0.0.1:8084 | it-tools |\n| karakeep.vpn.gram1.ru | http://127.0.0.1:3030 | karakeep |\n| linkding.vpn.gram1.ru | http://127.0.0.1:9091 | linkding |\n| minio.vpn.gram1.ru | http://127.0.0.1:9003 | minio |\n| ntfy.vpn.gram1.ru | http://127.0.0.1:8055 | ntfy |\n| searxng.vpn.gram1.ru | http://127.0.0.1:8888 | searxng |\n| syncthing.vpn.gram1.ru | http://127.0.0.1:8384 | syncthing |\n| alertmanager.vpn.gram1.ru | http://127.0.0.1:9093 | alertmanager |\n| prometheus.vpn.gram1.ru | http://127.0.0.1:9090 | prometheus |\n| homeassistant.vpn.gram1.ru | http://127.0.0.1:8123 | homeassistant |\n| npmplus.vpn.gram1.ru | https://[PRIVATE_IP]:81 | NPMplus admin, localhost-bound on edge |\n| proxmox.vpn.gram1.ru | https://[PRIVATE_IP]:8006 | pve01 Proxmox UI |\n| dns1.vpn.gram1.ru | http://[PRIVATE_IP]:8080 | dns1 AdGuard |\n| dns2.vpn.gram1.ru | http://[PRIVATE_IP]:8080 | dns2 AdGuard |\n| checks.vpn.gram1.ru | http://127.0.0.1:8015 | healthchecks |\n| vikunja.vpn.gram1.ru | http://127.0.0.1:8016 | vikunja |\n| bookstack.vpn.gram1.ru | http://127.0.0.1:8017 | bookstack |\n| pdf.vpn.gram1.ru | http://127.0.0.1:8018 | stirling-pdf |\n| jellyfin.vpn.gram1.ru | http://127.0.0.1:8019 | jellyfin |\n| audiobooks.vpn.gram1.ru | http://127.0.0.1:8020 | audiobookshelf |\n| books.vpn.gram1.ru | http://127.0.0.1:8021 | calibre-web |\n\n### Disabled / special routes\n- npm.gram1.ru exists in NPMplus but was observed disabled.\n- turn.gram1.ru is an AdGuard rewrite to [PRIVATE_IP] for Nextcloud Talk/TURN.\n- Router forwards TCP/UDP 3478 to Nextcloud [PRIVATE_IP].\n- Router forwards TCP/UDP 51820 for WireGuard on the Home bridge, not to edge-vm.\n\n### Critical dependency rules\n- If a public app is down, check in this order: DNS rewrite/upstream, NPMplus route/cert, upstream container/VM, app health file, backup/restore proof.\n- If a VPN route is down, check NetBird first, then NPMplus wildcard cert, then local upstream.\n- If DNS is broken, check dns1/dns2 AdGuard and unbound1/unbound2 before app containers.\n- If NPMplus is broken, use backup/restore evidence from npmplus-kuma-config and NPMplus DB backup.\n- If edge-vm is broken, VM130 full-image backup/offhost/restore proofs are authoritative.\n- If pve03 storage approaches WARN/CRIT, use pve03-staging-capacity health file.\n\n### Proof\n- Raw service dependency evidence index: 177_SERVICE_DEPENDENCY_MAP_RAW_EVIDENCE_INDEX.txt.\n- NPMplus route/cert rows: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt.\n- AdGuard/Unbound rewrites/upstreams: 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt.\n- Edge Docker container normalized map: 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt.\n- Proxmox VM/CT inventory: 121_PROXMOX_VM_CT_CONFIG_INVENTORY_REDACTED.txt.\n- Backup catalog: 141_REFERENCE_BACKUP_RESTORE_OFFHOST_CATALOG_BLOCK.txt.\n- Monitoring model: 147_REFERENCE_MONITORING_ALERTING_HEALTH_BLOCK.txt.\n\n## SERVICE_DEPENDENCY_MAP_FINAL_CLOSED_20260630\n\n- Service Dependency Map layer is closed.\n- Integrity proof: 180_SERVICE_DEPENDENCY_MAP_INTEGRITY_REVIEW.txt.\n- Secret scan proof: 179_SERVICE_DEPENDENCY_MAP_REFERENCE_PROOF_AND_SECRET_SCAN.txt.\n- Final closure proof: 181_SERVICE_DEPENDENCY_MAP_FINAL_CLOSURE.txt.\n## RUNBOOKS_RECOVERY_PROCEDURES_20260630\n\n### Universal operator preflight\n- Before every command, check /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md and the current 31_HOMELAB_REFERENCE.md.\n- Required visible markers: ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK.\n- If preflight fails, do not run the main action.\n- Do not use exit 1 in interactive SSH sessions.\n- Do not paste full terminal transcripts back into shell.\n- Do not print secrets.\n- Prefer short proof-producing commands over long nested quoting chains.\n\n### First triage order\n- Check current reference layer first.\n- Check latest proof file named by the relevant reference layer.\n- Check health files under /var/lib/homelab-health where applicable.\n- Check runtime state only after understanding the owning node, VM, container and proxy route.\n- Record every failed command or misleading proof in the error register before moving on.\n\n### Public application down\n- Start with Service Dependency Map.\n- Check DNS path: dns1/dns2 AdGuard, then unbound1/unbound2.\n- Check NPMplus route and certificate using NPMplus DB/proxy proof.\n- Check upstream app container or VM.\n- Check app-specific health, backup and restore proof.\n- Check external-canary and blackbox/Prometheus if route is public.\n- Do not change DNS, certificates or proxy routes without DB backup and proof.\n\n### DNS failure\n- dns1 is CT110 at [PRIVATE_IP].\n- dns2 is CT111 at [PRIVATE_IP].\n- unbound1 is CT112 at [PRIVATE_IP]:5335.\n- unbound2 is CT113 at [PRIVATE_IP]:5335.\n- AdGuard upstreams must point to local Unbound pair.\n- Rewrites include public routes for git, dozzle, paper, memos, photos, auth and backup to edge-vm.\n- turn.gram1.ru points to Nextcloud [PRIVATE_IP].\n- Use DNS/Ingress reference and proof files before editing configs.\n\n### Ingress / NPMplus failure\n- Edge VM is VM130 at [PRIVATE_IP].\n- NPMplus listens publicly on 80/443 and admin is bound to 127.0.0.1:81.\n- NPMplus DB is /opt/npmplus/npmplus/database.sqlite.\n- Before modifying certificates or proxy rows, create a DB backup.\n- Cloudflare token values must never be printed.\n- Use npmplus-kuma-config backup/offhost/restore proofs for recovery.\n\n### Edge VM failure\n- VM130 is the Docker runtime host.\n- First check Proxmox VM state and pve03 storage.\n- Then use VM130 full-image vzdump/offhost/restore proofs.\n- Current authoritative edge-vm backup/offhost/restore evidence is in backup catalog and VM130 proofs.\n- If restoring, verify exact archive, checksum and VMA/ZSTD integrity before booting replacement.\n\n### Proxmox / VM / CT restore\n- Primary nightly job: homelab-nightly-all at 03:30, zstd, snapshot.\n- Included VMIDs: 110,111,112,113,130,150,160.\n- Use backup catalog for latest known backup/offhost/restore evidence.\n- Do not infer offhost success from failed rsync.\n- Verify exact artifact, size and checksum where available.\n- For VM130 and VM160, use their dedicated closure proofs.\n\n### Monitoring / alerting failure\n- Prometheus is on edge-vm at 127.0.0.1:9090.\n- Alertmanager is on 127.0.0.1:9093.\n- Loki is on 127.0.0.1:3100.\n- Uptime Kuma is on 127.0.0.1:3001.\n- Gotify is on 127.0.0.1:8082.\n- ntfy is on 127.0.0.1:8055.\n- Healthchecks is on 127.0.0.1:8015.\n- Prometheus proof 153 is invalid and must not be used.\n- Use proof 154 and final closure 157 for Prometheus settled target state.\n\n### Backup dashboard / SLO interpretation\n- Runtime dashboard OK means current operational checks are green.\n- SLO backlog WARN means coverage improvement backlog, not necessarily runtime outage.\n- Do not merge SLO backlog WARN into runtime blockers unless explicitly promoted.\n- Use slo-coverage-backlog-index for backlog status.\n\n### Storage / capacity event\n- pve03 staging is monitored separately because it was observed at 77%.\n- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt.\n- WARN threshold: 80%.\n- CRIT threshold: 90%.\n- Timer: homelab-pve03-staging-capacity-health.timer.\n- Before cleanup, check retention policy and backup/offhost proof.\n- Do not use broad Docker prune by default.\n\n### Security / secret incident\n- Stop printing values immediately.\n- Identify whether the leak is value, file path, or false-positive text.\n- Rotate affected token/password if a value was exposed.\n- Create backup before DB/config mutation.\n- Re-run strict secret scan after rotation.\n- Record proof files and external revocation confirmation where applicable.\n- Cloudflare token and XenForo SMTP rotations already have closure proofs.\n\n### Forum / CodeVipe incident\n- forum-prod is VM160 at [PRIVATE_IP].\n- Access path is through pve02 using [SENSITIVE_PATH]\n- XenForo path: /var/www/codevipe/public.\n- SMTP host: mail.pvepro.ru on port 587 with STARTTLS semantics.\n- Do not enable smtpSsl=true blindly for port 587.\n- Do not use fragile nested PHP/base64 SMTP checkers.\n- Use XenForo SMTP rotation and auth proof files for current mail state.\n\n### Final evidence rules\n- Every remediation gets a numbered proof file.\n- Every reference block gets a proof and secret scan file.\n- Invalid proof files must be explicitly superseded, not silently ignored.\n- Reference closure requires integrity scan, secret scan and required-heading checks.\n## HOMELAB_REFERENCE_FINAL_OPERATOR_STATUS_20260630\n\n### Status\n- Current homelab reference is complete for the audited local infrastructure scope.\n- Final quality precheck passed.\n- Required headings are present.\n- Bad terminal/log marker scan is clean.\n- Strict secret scan is clean.\n- This is a reference/operator-status closure, not an archive/export.\n\n### Closed layers\n- Critical runtime tails closure.\n- Proxmox cluster, node, storage and VM/CT inventory.\n- DNS, ingress, NPMplus certificates and AdGuard/Unbound model.\n- Edge Docker stacks and container map.\n- Backup, restore, offhost and cloud backup catalog.\n- Monitoring, alerting, health dashboard and Prometheus correction.\n- Security, access and secrets operating model.\n- Storage, disk, SMART and capacity model with pve03 staging monitor.\n- Service Dependency Map.\n- Runbooks and recovery procedures.\n\n### Important superseded/invalid proofs\n- 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt is invalid and must not be used.\n- 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt is not sufficient by itself because symlink mode showed 777.\n- Use 154 for Prometheus settled targets and 163 for symlink-aware SSH target permissions.\n\n### Current authoritative final proof set\n- 185_HOMELAB_REFERENCE_FINAL_QUALITY_SCAN_PRECHECK.txt.\n- 184_RUNBOOKS_RECOVERY_REFERENCE_PROOF_AND_SECRET_SCAN.txt.\n- 181_SERVICE_DEPENDENCY_MAP_FINAL_CLOSURE.txt.\n- 176_STORAGE_FINAL_CLOSURE_WITH_PVE03_MONITOR.txt.\n- 165_SECURITY_ACCESS_SECRETS_FINAL_PROOF_AND_SECRET_SCAN.txt.\n- 157_MONITORING_PROMETHEUS_FINAL_CLOSURE.txt.\n- 142_BACKUP_RESTORE_OFFHOST_REFERENCE_PROOF_AND_SECRET_SCAN.txt.\n- 136_EDGE_DOCKER_STACKS_REFERENCE_PROOF_AND_SECRET_SCAN.txt.\n- 132_DNS_INGRESS_REFERENCE_PROOF_AND_SECRET_SCAN.txt.\n\n### Operator rule\n- Before every future command, continue checking both the error register and this reference file.\n- Required visible markers remain ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK.\n## HOMELAB_REFERENCE_DEEP_AUDIT_CLOSURE_20260630\n\n### Deep audit result\n- Error register and reference consistency audit passed.\n- Final proof files unresolved-marker audit passed.\n- Reference proof-link audit passed.\n- All referenced proof files exist.\n- No unresolved final REVIEW_REQUIRED, PROOF_MISSING, HEADING_MISSING, BAD_MARKER, STRICT_SECRET, RULE_CHECK_FAIL, SyntaxError or PERM_BAD markers remained in final proof set.\n- Invalid/superseded proofs are explicitly documented and must not be used as authoritative closure evidence.\n\n### Authoritative deep audit proofs\n- 189_DEEP_ERROR_REGISTER_REFERENCE_CONSISTENCY_AUDIT.txt.\n- 190_DEEP_PROOF_FILES_UNRESOLVED_MARKER_AUDIT.txt.\n- 191_DEEP_REFERENCE_PROOF_LINK_AUDIT.txt.\n\n### Scope statement\n- This closes the current audited local homelab reference scope.\n- External systems can still receive separate dedicated passports if the scope is expanded later.\n\n## NEXTCLOUD_VM150_MAIL_CLOUD_BACKUP_20260630\n- VM150 Nextcloud Mail-cloud backup is installed on pve01.\n- Timer: homelab-mail-cloud-nextcloud-vm-upload.timer.\n- Health file: /var/lib/homelab-health/mail-cloud-nextcloud-vm.txt.\n- Current proof health: STATUS=OK, MODE=recurring-chunked, PARTS=14, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no.\n- Proof: 192_NEXTCLOUD_VM150_MAIL_CLOUD_CHUNKED_PROOF.txt.\n\n## ROUTER_RECURRING_BACKUP_BLOCKER_20260630\n- Router recurring backup from pve01 is not installed yet.\n- Reason: pve01 cannot reach router management ports 2222, 5080, 5083 or 2323 on [PRIVATE_IP].\n- Current router backup state remains reference/self-test based until router management access is allowed or startup-config is exported by another approved path.\n- Do not run bash/POSIX automation against Netcraze CLI; use router CLI syntax only.\n- Proof: 195_ROUTER_RECURRING_BACKUP_BLOCKED_BY_MGMT_ACL_PROOF.txt.\n\n## EDGE_GAP_RESTORE_DRYRUN_CLOSURE_20260630\n- Restore dry-run gaps closed for Actual Budget, Home Assistant, Karakeep, Linkding, Mealie, n8n and Observability config.\n- Production data was not restored or modified; checks used temporary extraction, SHA verification, tar listing/extraction and SQLite integrity where applicable.\n- Current observed restore health: all 7 STATUS=OK with RESTORE_ATTEMPTED=YES and SECRET_PRINTED=no.\n- Proof: 197_EDGE_GAP_RESTORE_DRYRUN_CLOSURE_PROOF.txt.\n\n## P2_SMALL_STACKS_BACKUP_RESTORE_CLOSURE_20260630\n- P2 small-stack backup and restore dry-run is installed on edge-vm.\n- Covered paths: Syncthing config, ntfy cache/data, MinIO data, Homebox data and Node-RED data.\n- Timer: homelab-p2-small-stacks-backup-restore.timer.\n- Current observed health: STATUS=OK, BACKUP_PATH_COUNT=6, RESTORE_ATTEMPTED=YES, SQLITE_INTEGRITY_OK=yes, SECRET_PRINTED=no.\n- Proof: 199_P2_SMALL_STACKS_BACKUP_RESTORE_CLOSURE_PROOF.txt.\n\n## BACKUP_CLOUD_AND_RESTORE_FINAL_CLOSURE_20260630\n- Backup/cloud/restore improvement pass is closed with runtime dashboard STATUS=OK and NOT_OK=0.\n- VM150 Nextcloud now has Mail-cloud chunked backup with download verification.\n- Restore dry-run gaps closed for Actual Budget, Home Assistant, Karakeep, Linkding, Mealie, n8n and Observability config.\n- P2 small stacks now have backup and restore dry-run coverage for Syncthing config, ntfy, MinIO, Homebox and Node-RED.\n- Router recurring backup remains blocked by management ACL and is documented as blocker, not silently closed.\n- Proof: 201_BACKUP_CLOUD_AND_RESTORE_FINAL_CLOSURE_PROOF.txt.\n\n## ROUTER_STARTUP_CONFIG_MAIL_CLOUD_MANUAL_20260630\n- Router startup-config manual backup is stored in Mail-cloud crypt remote.\n- Source file content is not printed in proofs or reference.\n- Health file: /var/lib/homelab-health/router-startup-config-mail-cloud.txt.\n- Current observed health: STATUS=OK, UPLOAD_OK=yes, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no.\n- Recurring router backup is still blocked until pve01 can reach router management ports.\n- Proof: 203_ROUTER_STARTUP_CONFIG_MAIL_CLOUD_MANUAL_PROOF.txt.\n\n## ROUTER_RUNNING_CONFIG_MAIL_CLOUD_RECURRING_20260630\n- Router running-config recurring Mail-cloud backup is installed on pve01.\n- Timer: homelab-router-running-config-mail-cloud.timer.\n- Health file: /var/lib/homelab-health/router-running-config-mail-cloud.txt.\n- Current observed health: STATUS=OK, SSH_RC=0, LOOKS_CONFIG=yes, UPLOAD_OK=yes, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no.\n- Manual startup-config Mail-cloud backup also exists as separate proof.\n- Proof: 205_ROUTER_RUNNING_CONFIG_MAIL_CLOUD_RECURRING_PROOF.txt.\n\n## POST_BACKUP_PASS_SLO_RECONCILIATION_20260630\n- Post backup/cloud/restore pass SLO reconciliation is closed.\n- Runtime backup dashboard remains STATUS=OK with NOT_OK=0.\n- Restore index remains STATUS=OK with RESTORE_PROOFS=16/16.\n- VM150 Mail-cloud backup, router manual/recurring Mail-cloud backups, P2 small-stacks and 7 edge restore dry-runs are all confirmed OK.\n- Existing backup-restore-coverage-slo.txt is historical model output from 20260625 and does not represent current runtime blocker state.\n- Current reconciliation health: STATUS=OK, RUNTIME_BLOCKER=NO, SECRET_PRINTED=no.\n- Proof: 208_POST_BACKUP_PASS_SLO_RECONCILIATION_PROOF.txt.\n\n## POST_BACKUP_PASS_ALERTING_20260630\n- Prometheus alerting for post-backup-pass health files is installed and loaded.\n- Rule file: /opt/stacks/observability-lite/prometheus/rules/homelab-post-backup-pass-alerts.yml.\n- Alerts: HomelabPostBackupPassHealthNotOk and HomelabPostBackupPassHealthStale.\n- Covered checks include VM150 Mail-cloud, router startup/running config Mail-cloud, P2 small-stacks and seven restore dry-run health files.\n- Current proof confirms rule validation, Prometheus API visibility and up targets.\n- Proof: 210_POST_BACKUP_PASS_ALERTING_PROOF.txt.\n\n## POST_BACKUP_PASS_ALERTS_NOT_FIRING_20260630\n- Post-backup-pass Prometheus alert rules are loaded and currently not firing.\n- Corrected proof uses direct PromQL ALERTS query for firing state, not grep over rule names.\n- Alerts checked: HomelabPostBackupPassHealthNotOk and HomelabPostBackupPassHealthStale.\n- Proof: 213_POST_BACKUP_PASS_ALERTS_NOT_FIRING_CORRECTED_PROOF.txt.\n\n## MAIL_CLOUD_CAPACITY_RETENTION_20260630\n- Mail-cloud capacity and backup directory inventory was checked after post-backup-pass closure.\n- VM150 Nextcloud, router startup/running config, P0 critical and edge-vm backup directories are present on crypt remotes.\n- Observed capacity is sufficient: largest used remote was pve03-mail-03-crypt around 65 GiB of 1 TiB.\n- Retention shape observed: P0 critical has 10 visible dirs with keep 14, edge-vm has 3 visible dirs with keep 4, new VM150/router jobs have initial dirs.\n- Proof: 216_MAIL_CLOUD_CAPACITY_RETENTION_PROOF.txt.\n\n## ROUTER_BACKUP_SECRET_PERMISSION_20260630\n- Router recurring backup uses a dedicated routerbackup credential file on pve01.\n- Secret file path: /var/lib/homelab-private/secrets/routerbackup.pass.\n- File content must never be printed; only mode/owner/size may be checked.\n- Current observed permission target: root-owned mode 600.\n- Proof: 218_ROUTER_BACKUP_SECRET_PERMISSION_PROOF.txt.\n\n## NEW_BACKUP_UNITS_SYSTEMD_INVENTORY_20260630\n- New backup/restore systemd units and timers were inventoried after post-backup-pass closure.\n- pve01 units checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup.\n- edge-vm units checked: P2 small-stacks backup/restore, backup dashboard and restore drill index.\n- Proof records LoadState, ActiveState, UnitFileState and failed-unit counts for pve01 and edge-vm.\n- Proof: 222_NEW_BACKUP_UNITS_SYSTEMD_INVENTORY_PROOF.txt.\n\n## POST_BACKUP_PASS_PROM_HEALTH_COVERAGE_20260630\n- Prometheus health coverage was checked for the post-backup-pass checks.\n- Covered checks: VM150 Mail-cloud, router startup/running config, P2 small-stacks and seven restore dry-runs.\n- Proof records health status and age from Prometheus without printing credential values.\n- Proof: 224_POST_BACKUP_PASS_PROM_HEALTH_COVERAGE_PROOF.txt.\n\n## POST_BACKUP_PASS_AUDIT_INDEX_20260630\n- Audit proof manifest was generated for post-backup-pass evidence files 192-225.\n- Manifest records file names, sha256 hashes, count and missing-number check.\n- Proof: 226_POST_BACKUP_PASS_AUDIT_INDEX_PROOF.txt.\n\n## POST_BACKUP_PASS_EXTENDED_FINAL_SNAPSHOT_20260630\n- Extended post-backup-pass final snapshot was created after capacity, retention, unit inventory, Prometheus coverage and audit index checks.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-227.\n- Proof: 228_POST_BACKUP_PASS_EXTENDED_FINAL_SNAPSHOT_PROOF.txt.\n\n## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_229_20260630\n- Final audit manifest was generated after the extended final snapshot.\n- Manifest covers proof numbers 192-229, including snapshot and reference-scan proofs.\n- Manifest records file names, sha256 hashes, count and missing-number check.\n- Proof: 230_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_229_PROOF.txt.\n\n## NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_20260630\n- New backup/restore unit files and executable script paths were inventoried and hashed.\n- pve01 services checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup.\n- edge-vm services checked: P2 small-stacks backup/restore, backup dashboard and restore drill index.\n- Proof records FragmentPath, ExecStart path, mode/owner/size and sha256 hashes without printing script contents.\n- Proof: 232_NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_PROOF.txt.\n\n## NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_CORRECTED_20260630\n- Corrected integrity proof was generated because one edge-vm executable required sudo for sha256sum.\n- Corrected proof records hashes for pve01 and edge-vm unit fragments and executable scripts without printing script contents.\n- Proof: 234_NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_CORRECTED_PROOF.txt.\n\n## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_235_20260630\n- Final audit manifest was regenerated after corrected unit/script integrity proof.\n- Manifest covers proof numbers 192-235, including corrected integrity proof 234/235.\n- Manifest records file names, sha256 hashes, count and missing-number check.\n- Proof: 236_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_235_PROOF.txt.\n\n## NEW_BACKUP_TIMERS_INTEGRITY_20260630\n- New backup/restore timer unit files were inventoried and hashed.\n- pve01 timers checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup.\n- edge-vm timers checked: P2 small-stacks backup/restore, backup dashboard and restore drill index.\n- Proof records FragmentPath, mode/owner/size, sha256 hashes and next-elapse metadata without printing sensitive values.\n- Proof: 238_NEW_BACKUP_TIMERS_INTEGRITY_PROOF.txt.\n\n## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_239_20260630\n- Final audit manifest was regenerated after timer integrity proof.\n- Manifest covers proof numbers 192-239 and records file names, sha256 hashes, count and missing-number check.\n- Proof: 240_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_239_PROOF.txt.\n\n## POST_BACKUP_PASS_FINAL_SNAPSHOT_AFTER_TIMERS_20260630\n- Final snapshot was created after timer integrity and final audit manifest 192-239.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-241.\n- Proof: 242_POST_BACKUP_PASS_FINAL_SNAPSHOT_AFTER_TIMERS_PROOF.txt.\n\n## ASSISTANT_COMMAND_BATCHING_RULE_20260630\n- Operator preference: combine related homelab commands into fewer package-style runs to reduce slow repetitive manual work.\n- Applies to proof generation, reference updates, validation scans, hash/index refreshes and snapshot steps when safe.\n- Mandatory preflight and sensitive-output hygiene still take priority.\n\n## POST_BACKUP_PASS_AUDIT_INDEX_AFTER_BATCHING_RULE_192_245_20260630\n- Audit manifest was regenerated after assistant command batching rule was recorded.\n- Manifest covers proof numbers 192-245 and records file names, sha256 hashes, count and missing-number check.\n- Proof: 246_POST_BACKUP_PASS_AUDIT_INDEX_AFTER_BATCHING_RULE_192_245_PROOF.txt.\n\n## POST_BACKUP_PASS_SNAPSHOT_AFTER_BATCHING_RULE_20260630\n- Snapshot was created after assistant command batching rule and audit manifest 192-245.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-247.\n- Proof: 248_POST_BACKUP_PASS_SNAPSHOT_AFTER_BATCHING_RULE_PROOF.txt.\n\n## ALERTMANAGER_ROUTING_AND_CONFIG_20260630\n- Alertmanager routing/config health was checked after post-backup-pass alert rules were installed.\n- Proof records container presence, config validation, readiness endpoint and sanitized route/receiver metadata.\n- Sensitive receiver values and URLs are intentionally not printed.\n- Proof: 250_ALERTMANAGER_ROUTING_AND_CONFIG_PROOF.txt.\n\n## AUDIT_INDEX_192_251_20260630\n- Audit manifest was regenerated after Alertmanager routing/config proof.\n- Manifest covers proof numbers 192-251 with count and missing-number check.\n- Proof: 252_AUDIT_INDEX_192_251_PROOF.txt.\n\n## RCLONE_CONFIG_PERMISSION_AND_REMOTE_INVENTORY_20260630\n- rclone config permissions and crypt remote inventory were checked without printing config contents.\n- Proof records config owner/mode/size, remote names and capacity summaries for active Mail-cloud crypt remotes.\n- Proof: 254_RCLONE_CONFIG_PERMISSION_AND_REMOTE_INVENTORY_PROOF.txt.\n\n## AUDIT_INDEX_192_255_20260630\n- Audit manifest was regenerated after rclone config/remote inventory proof.\n- Manifest covers proof numbers 192-255 with count and missing-number check.\n- Proof: 256_AUDIT_INDEX_192_255_PROOF.txt.\n\n## CURRENT_OPERATIONAL_ROLLUP_20260630\n- Current operational rollup was captured after rclone config/remote inventory proof.\n- Rollup records failed-unit counts, next timers, Mail-cloud capacity, Prometheus target count, post-backup health status/age, post-backup firing alerts and Alertmanager readiness.\n- Proof: 258_CURRENT_OPERATIONAL_ROLLUP_PROOF.txt.\n\n## AUDIT_INDEX_192_259_20260630\n- Audit manifest was regenerated after current operational rollup.\n- Manifest covers proof numbers 192-259 with count and missing-number check.\n- Proof: 260_AUDIT_INDEX_192_259_PROOF.txt.\n\n## SNAPSHOT_AFTER_CURRENT_OPERATIONAL_ROLLUP_20260630\n- Snapshot was created after current operational rollup and audit index 192-259.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-261.\n- Proof: 262_SNAPSHOT_AFTER_CURRENT_OPERATIONAL_ROLLUP_PROOF.txt.\n\n## AUDIT_INDEX_192_263_20260630\n- Audit manifest was regenerated after snapshot following current operational rollup.\n- Manifest covers proof numbers 192-263 with count and missing-number check.\n- Proof: 264_AUDIT_INDEX_192_263_PROOF.txt.\n\n## NEW_BACKUP_SERVICE_RUNTIME_RESULT_20260630\n- Runtime result metadata was captured for new backup/restore service units.\n- pve01 services checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup.\n- edge-vm services checked: P2 small-stacks backup/restore, backup dashboard and restore drill index.\n- Proof records LoadState, ActiveState, Result, ExecMainStatus and restart counters without printing sensitive values.\n- Proof: 266_NEW_BACKUP_SERVICE_RUNTIME_RESULT_PROOF.txt.\n\n## AUDIT_INDEX_192_267_20260630\n- Audit manifest was regenerated after new backup service runtime-result proof.\n- Manifest covers proof numbers 192-267 with count and missing-number check.\n- Proof: 268_AUDIT_INDEX_192_267_PROOF.txt.\n\n## NEW_BACKUP_UNITS_JOURNAL_ERROR_SCAN_20260630\n- Journal warning/error counters were captured for new backup/restore units and timers over the last 24 hours.\n- Proof records only counts, not journal message bodies, to avoid sensitive-output risk.\n- Proof: 270_NEW_BACKUP_UNITS_JOURNAL_ERROR_SCAN_PROOF.txt.\n\n## AUDIT_INDEX_192_271_20260630\n- Audit manifest was regenerated after journal error scan proof.\n- Manifest covers proof numbers 192-271 with count and missing-number check.\n- Proof: 272_AUDIT_INDEX_192_271_PROOF.txt.\n\n## NEW_BACKUP_UNITS_JOURNAL_ERROR_TRIAGE_20260630\n- Journal warning/error triage was captured after warning/error counters were non-zero.\n- Proof records per-unit warning/error counts and redacted journal fragments.\n- Proof: 274_NEW_BACKUP_UNITS_JOURNAL_ERROR_TRIAGE_PROOF.txt.\n\n## AUDIT_INDEX_192_275_20260630\n- Audit manifest was regenerated after journal error triage proof.\n- Manifest covers proof numbers 192-275 with count and missing-number check.\n- Proof: 276_AUDIT_INDEX_192_275_PROOF.txt.\n\n## VM150_MAIL_CLOUD_JOURNAL_NOISE_CLASSIFICATION_20260630\n- VM150 Mail-cloud upload journal warnings/errors were classified after journal triage found non-zero counters.\n- Current service result, Prometheus health and firing-alert status are used to decide whether journal noise is blocking.\n- Proof includes redacted journal indicators only, not secrets.\n- Proof: 278_VM150_MAIL_CLOUD_JOURNAL_NOISE_CLASSIFICATION_PROOF.txt.\n\n## AUDIT_INDEX_192_279_20260630\n- Audit manifest was regenerated after VM150 Mail-cloud journal-noise classification.\n- Manifest covers proof numbers 192-279 with count and missing-number check.\n- Proof: 280_AUDIT_INDEX_192_279_PROOF.txt.\n\n## SNAPSHOT_AFTER_VM150_JOURNAL_NOISE_CLASSIFICATION_20260630\n- Snapshot was created after VM150 Mail-cloud journal noise was classified as non-blocking historical noise.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-281.\n- Proof: 282_SNAPSHOT_AFTER_VM150_JOURNAL_NOISE_CLASSIFICATION_PROOF.txt.\n\n## AUDIT_INDEX_192_283_20260630\n- Audit manifest was regenerated after snapshot following VM150 journal-noise classification.\n- Manifest covers proof numbers 192-283 with count and missing-number check.\n- Proof: 284_AUDIT_INDEX_192_283_PROOF.txt.\n\n## POST_BACKUP_PASS_CLOSURE_SUMMARY_20260630\n- Post-backup-pass closure summary was generated after VM150 journal-noise classification.\n- Summary checks backup SLO closure, alert rules, firing-alert status, Prometheus health coverage, failed-unit state, Alertmanager readiness, rclone config hygiene, journal-noise classification and audit index completeness.\n- Proof: 286_POST_BACKUP_PASS_CLOSURE_SUMMARY_PROOF.txt.\n\n## SNAPSHOT_AFTER_POST_BACKUP_PASS_CLOSURE_SUMMARY_20260630\n- Snapshot was created after post-backup-pass closure summary.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-287.\n- Proof: 288_SNAPSHOT_AFTER_POST_BACKUP_PASS_CLOSURE_SUMMARY_PROOF.txt.\n\n## AUDIT_INDEX_192_289_20260630\n- Audit manifest was regenerated after post-backup-pass closure summary snapshot.\n- Manifest covers proof numbers 192-289 with count and missing-number check.\n- Proof: 290_AUDIT_INDEX_192_289_PROOF.txt.\n\n## TOMORROW_FIRST_RUN_VERIFICATION_PLAN_20260630\n- First scheduled-run verification should be done after 2026-07-01 08:15 MSK.\n- Verify P2 small-stacks, router running-config Mail-cloud and VM150 Nextcloud Mail-cloud first automatic runs.\n- Also verify service results, health status/age, Mail-cloud directories, dashboard, restore index, Prometheus firing alerts and Alertmanager readiness.\n- Proof: 292_TOMORROW_FIRST_RUN_VERIFICATION_PLAN_PROOF.txt.\n\n## AUDIT_INDEX_192_293_20260630\n- Audit manifest was regenerated after tomorrow first-run verification plan.\n- Manifest covers proof numbers 192-293 with count and missing-number check.\n- Proof: 294_AUDIT_INDEX_192_293_PROOF.txt.\n\n## END_OF_DAY_POST_BACKUP_PASS_SNAPSHOT_20260630\n- End-of-day snapshot was created after post-backup-pass closure summary and tomorrow first-run verification plan.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range before this snapshot: 192-295.\n- Proof: 296_END_OF_DAY_POST_BACKUP_PASS_SNAPSHOT_PROOF.txt.\n\n## AUDIT_INDEX_192_297_20260630\n- Final end-of-day audit manifest was generated after post-backup-pass snapshot.\n- Manifest covers proof numbers 192-297 with count and missing-number check.\n- Proof: 298_AUDIT_INDEX_192_297_PROOF.txt.\n\n## HOME_PORTAL_DISCOVERY_20260630\n- Home portal discovery started for https://home.gram1.ru/.\n- Goal: make the portal show and open all web services, including external service cards such as NetBird and mail.\n- Discovery proof records DNS, HTTPS/TLS headers, reverse-proxy candidates, compose files, domain references and homepage config candidates without printing secrets.\n- Proof: 300_HOME_PORTAL_DISCOVERY_PROOF.txt.\n\n## HOME_PORTAL_CONFIG_AND_SERVICE_INVENTORY_20260630\n- Home portal config and service inventory was collected for https://home.gram1.ru/.\n- Inventory records portal containers, config files, redacted card lines, compose web hints, domain references and common subdomain probes.\n- Proof: 302_HOME_PORTAL_CONFIG_AND_SERVICE_INVENTORY_PROOF.txt.\n\n## HOME_PORTAL_GAP_ANALYSIS_20260630\n- Home portal gap analysis was collected for gethomepage/homepage behind npmplus.\n- Proof records Homepage config hashes, redacted current cards/bookmarks, current URLs, npmplus route lines, local web-port probes and known service container candidates.\n- Proof: 304_HOME_PORTAL_GAP_ANALYSIS_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_305_20260630\n- Home portal audit manifest was generated for proof numbers 300-305.\n- Proof: 306_HOME_PORTAL_AUDIT_INDEX_300_305_PROOF.txt.\n\n## HOME_PORTAL_CARD_AND_API_ERROR_ANALYSIS_20260630\n- Home portal card/API-error analysis was collected before editing Homepage.\n- Proof records current card URLs/titles, redacted widget config, redacted Homepage log errors, NPMPlus internal route files/routes and candidate public/VPN cards.\n- Proof: 308_HOME_PORTAL_CARD_AND_API_ERROR_ANALYSIS_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_309_20260630\n- Home portal audit manifest was regenerated after card/API-error analysis.\n- Manifest covers proof numbers 300-309 with count and missing-number check.\n- Proof: 310_HOME_PORTAL_AUDIT_INDEX_300_309_PROOF.txt.\n\n## HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_APPLY_20260630\n- Homepage services.yaml was backed up, duplicate VPN cards were removed where a non-VPN card existed, and missing web cards for NetBird, Mail and Webmail were added without VPN suffix in the card names.\n- SMTP/IMAP were intentionally not added as cards because they are protocol endpoints, not web UI cards.\n- Homepage container was restarted and portal/card URLs were checked.\n- Proof: 312_HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_APPLY_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_313_20260630\n- Home portal audit manifest was regenerated after duplicate-card cleanup and external-card addition.\n- Manifest covers proof numbers 300-313 with count and missing-number check.\n- Proof: 314_HOME_PORTAL_AUDIT_INDEX_300_313_PROOF.txt.\n\n## HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_CORRECTED_APPLY_20260630\n- Corrected Homepage apply removed duplicate VPN cards where a non-VPN card existed and added NetBird, Mail and Webmail cards with non-VPN names.\n- The added card URLs use currently reachable vpn.gram1.ru web endpoints because public netbird/mail/webmail hosts did not resolve/open during analysis.\n- SMTP/IMAP were not added as cards because they are protocol endpoints, not web UIs.\n- Proof: 316_HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_CORRECTED_APPLY_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_317_20260630\n- Home portal audit manifest was regenerated after corrected duplicate-card cleanup and external-card addition.\n- Manifest covers proof numbers 300-317 with count and missing-number check.\n- Proof: 318_HOME_PORTAL_AUDIT_INDEX_300_317_PROOF.txt.\n\n## HOMELAB_COMMAND_SAFETY_HARDENING_20260630\n- Command safety rule strengthened after the home portal base64 apply failure.\n- Opaque generated payloads are not acceptable for homelab changes; commands must be transparent, bounded, and validated before modification.\n- Success requires content-specific checks in addition to service/runtime checks.\n- Proof: 320_HOMELAB_COMMAND_SAFETY_HARDENING_RULE_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_321_20260630\n- Home portal audit manifest was regenerated after command-safety hardening rule.\n- Manifest covers proof numbers 300-321 with count and missing-number check.\n- Proof: 322_HOME_PORTAL_AUDIT_INDEX_300_321_PROOF.txt.\n\n## HOME_PORTAL_POST_APPLY_VALIDATION_AND_API_TRIAGE_20260630\n- Home portal was validated after duplicate cleanup and external-card addition.\n- Proof checks portal HTTP status, Homepage container state, required cards, duplicate VPN card pairs, current card URLs, redacted API-error logs and widget configuration.\n- Proof: 324_HOME_PORTAL_POST_APPLY_VALIDATION_AND_API_TRIAGE_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_325_20260630\n- Home portal audit manifest was regenerated after post-apply validation and API triage.\n- Manifest covers proof numbers 300-325 with count and missing-number check.\n- Proof: 326_HOME_PORTAL_AUDIT_INDEX_300_325_PROOF.txt.\n\n## HOME_PORTAL_API_ERROR_ROOT_CAUSE_20260630\n- Homepage API error root-cause analysis was collected after the UI showed API errors.\n- Current evidence points to the Open-Meteo/weather widget timing out from Homepage, separate from service cards.\n- HTTP 200 for the portal is not sufficient for UI closure when Homepage logs still contain API errors.\n- Proof: 328_HOME_PORTAL_API_ERROR_ROOT_CAUSE_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_329_20260630\n- Home portal audit manifest was regenerated after API-error root-cause analysis.\n- Manifest covers proof numbers 300-329 with count and missing-number check.\n- Proof: 330_HOME_PORTAL_AUDIT_INDEX_300_329_PROOF.txt.\n\n## HOME_PORTAL_DISABLE_OPENMETEO_WIDGET_20260630\n- Open-Meteo/weather widget was disabled after root-cause analysis showed Homepage API errors from api.open-meteo.com timeouts.\n- This keeps the portal focused on service-launcher cards and prevents external weather API failures from causing UI API errors.\n- Portal closure now requires current Homepage API-error logs to be zero after restart/reload.\n- Proof: 332_HOME_PORTAL_DISABLE_OPENMETEO_WIDGET_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_333_20260630\n- Home portal audit manifest was regenerated after disabling Open-Meteo/weather widget.\n- Manifest covers proof numbers 300-333 with count and missing-number check.\n- Proof: 334_HOME_PORTAL_AUDIT_INDEX_300_333_PROOF.txt.\n\n## HOME_PORTAL_LINK_OPEN_AUDIT_20260630\n- Home portal card links were audited after Open-Meteo/weather widget was disabled.\n- Audit extracts current Homepage href URLs and checks whether each opens with an acceptable HTTP status.\n- Acceptable statuses include success, redirects and auth-gated 401/403 pages because those still indicate a reachable web service.\n- Proof: 336_HOME_PORTAL_LINK_OPEN_AUDIT_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_337_20260630\n- Home portal audit manifest was regenerated after link-open audit.\n- Manifest covers proof numbers 300-337 with count and missing-number check.\n- Proof: 338_HOME_PORTAL_AUDIT_INDEX_300_337_PROOF.txt.\n\n## HOME_PORTAL_EXPLICIT_VPN_DUPLICATE_CLEANUP_CORRECTED_20260630\n- Explicit VPN duplicate cards were removed from Homepage with a short transparent perl edit: Homepage VPN, Authentik VPN, Gitea VPN and Uptime Kuma VPN.\n- Old router URL [PRIVATE_IP] was replaced with [PRIVATE_IP] where present.\n- Proof: 342_HOME_PORTAL_EXPLICIT_VPN_DUPLICATE_CLEANUP_CORRECTED_PROOF.txt.\n\n## HOME_PORTAL_REMAINING_DUPLICATE_OCCURRENCE_ANALYSIS_20260630\n- Remaining Homepage duplicate/VPN/router occurrences were inspected after corrected cleanup still reported REVIEW.\n- Proof records exact redacted file/line occurrences before another edit.\n- Proof: 344_HOME_PORTAL_REMAINING_DUPLICATE_OCCURRENCE_ANALYSIS_PROOF.txt.\n\n## HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RECHECK_20260630\n- Homepage duplicate cleanup was rechecked against active config files only, excluding logs and backup files.\n- Previous broad grep included historical logs/backups and therefore overstated remaining active duplicates.\n- Proof: 346_HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RECHECK_PROOF.txt.\n\n## HOME_PORTAL_ACTIVE_VPN_DUPLICATES_FINAL_CLEANUP_20260630\n- Active Homepage services/bookmarks files were cleaned from semantic VPN duplicate cards and old router URL references.\n- Removed targets: Homepage VPN, Authentik VPN, Gitea VPN and Uptime Kuma VPN, plus matching vpn.gram1.ru duplicate URLs.\n- Proof: 350_HOME_PORTAL_ACTIVE_VPN_DUPLICATES_FINAL_CLEANUP_PROOF.txt.\n\n## HOME_PORTAL_NPMPLUS_DEFAULT_ROUTE_ANALYSIS_20260630\n- Home portal NetBird/Mail/Webmail cards were investigated after user saw the NPMPlus default page from client [PRIVATE_IP].\n- Analysis checks URL body fingerprints, NPMPlus route config matches, service container candidates and Homepage card lines.\n- Proof: 356_HOME_PORTAL_NPMPLUS_DEFAULT_ROUTE_ANALYSIS_PROOF.txt.\n\n## HOME_PORTAL_CURRENT_STATE_20260630\n- Active Homepage URL: https://home.gram1.ru.\n- Homepage container is running and portal HTTP check returned 200 after latest changes.\n- NetBird card points to https://nb.pvepro.ru.\n- Mail card points to https://mail.pvepro.ru.\n- Webmail card was removed; no separate Webmail card is currently configured.\n- Router bookmark points to http://[PRIVATE_IP]:5080 for operator client LAN [PRIVATE_IP].\n- Homepage services.yaml was updated via YAML-aware edit for siteMonitor fields.\n- Current active services.yaml has SITEMONITOR_COUNT=43.\n- Latest validation before this reference update: YAML_ERRORS_LAST_5M=0, HOME_HTTP_CODE=200, HOMEPAGE_RUNNING=true.\n- Do not use sed-only insertion for siteMonitor; use YAML-aware editing and validate YAML before restart.\n- Related active-state checks: NB_LINK_ACTIVE=yes, MAIL_LINK_ACTIVE=yes, WEBMAIL_ACTIVE=no, ROUTER_192_168_1_1_ACTIVE=yes.\n\n## HOME_PORTAL_DIRECT_LINK_FIX_20260630\n- Direct Homepage link correction requested by operator.\n- NetBird: https://nb.pvepro.ru.\n- Mail: https://mail.pvepro.ru.\n- Webmail card removed.\n- Router restored to http://[PRIVATE_IP]:5080.\n\n## HOME_PORTAL_YAML_AWARE_SITEMONITOR_20260630\n- siteMonitor fields were added using YAML-aware PyYAML editing, not line-based sed insertion.\n- Added siteMonitor to 43 service cards.\n- Excluded cards: Homepage, NPMplus, Router and Public Domain.\n- YAML validation passed before restart: YAML_VALIDATE_RC=0.\n- Homepage restarted successfully and reported YAML_ERRORS=0.\n\n## HOMELAB_CLUSTER_BACKLOG_20260630\n- Cluster backlog was created at /etc/pve/HOMELAB_CLUSTER_BACKLOG.md.\n- PBS is intentionally out of scope; backup strategy remains Mail-cloud based.\n- First next action: finish Homepage final validation.\n\n## HOMEPAGE_BACKUP_GAP_CLASSIFICATION_20260630\n- Homepage backup coverage gap classification file created at /etc/pve/HOMEPAGE_BACKUP_GAP_CLASSIFICATION.md.\n- It defines closure requirements for the 13 remaining Homepage backup coverage gap/special-review rows.\n- PBS is intentionally out of scope; closure must use Mail-cloud backups or explicit stateless/config-only classification.\n\n## HOMEPAGE_EXISTING_BACKUP_EVIDENCE_CLOSURE_20260630\n- Homepage backup coverage matrix started closing existing-evidence rows.\n- Router maps to existing Mail-cloud startup/running-config backup proofs when ROUTER_EXISTING_EVIDENCE=yes.\n- MinIO Console maps to existing P2 small-stacks backup/restore proof when MINIO_EXISTING_EVIDENCE=yes.\n- Proof: 364_HOMEPAGE_EXISTING_BACKUP_EVIDENCE_CLOSURE_PROOF.txt.\n\n## HOME_EXTERNAL_CLOUDFLARE_ALEXHOST_20260630\n- Homepage External group includes Cloudflare card: https://dash.cloudflare.com.\n- Homepage External group includes AlexHost Billing card: https://bill.alexhost.com/.\n- Both cards have siteMonitor enabled for green status dots.\n- YAML validation passed, Homepage returned HTTP 200, container running, and no Homepage YAML errors were seen after restart.\n- Proof: 368_HOME_EXTERNAL_CLOUDFLARE_ALEXHOST_PROOF.txt.\n\n## HOME_EXTERNAL_RELAY_REMOVED_20260630\n- Homepage External card/link for relay.pvepro.ru was removed because relay.pvepro.ru had DNS but no reachable HTTP/HTTPS endpoint from edge-vm.\n- Proof: 382_HOME_EXTERNAL_RELAY_REMOVE_PROOF.txt.\n\n## ROUTER_CLI_PERMISSION_LIMIT_20260630\n- Netcraze routerbackup SSH access is read-only for backup: show running-config works, but ACL/config commands are denied.\n- Homepage Moscow Router red-dot fix requires router ACL/Web UI/admin-level change, not routerbackup.\n- Relevant proofs: 391, 393, 394, 395.\n\n## ROUTER_CLI_PROOF_REPAIR_20260630\n- Error register updated for blank proof summary extraction in 395.\n- Corrected derived counts are recorded in 396_ROUTER_CLI_ERROR_REGISTER_REPAIR_PROOF.txt.\n\n## HOME_EXTERNAL_NETBIRD_ALEXHOST_NORMALIZE_20260630\n- Homepage External NetBird and AlexHost Billing href/siteMonitor URLs normalized with trailing slash.\n- Homepage container node checks returned HTTP 200 for both URLs.\n- Proof: 397_HOME_EXTERNAL_NETBIRD_ALEXHOST_NORMALIZE_PROOF.txt.\n\n## ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701\n- Moscow Router ACL is restored and correct after manual Web UI edits.\n- Remaining blocker for Homepage green dot is HTTP/HTTPS 403 from router web service to edge-vm/Bridge1, not network reachability.\n- Relevant proofs: 399, 400, 401, 402.\n\n## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_20260701\n- Moscow Router Homepage card href: http://[PRIVATE_IP]:5080.\n- Moscow Router Homepage siteMonitor: http://[PRIVATE_IP]:18083/router-moscow.\n- Health endpoint service on edge-vm: homelab-router-moscow-health-http.service.\n- Endpoint semantics: returns 200 when TCP to router [PRIVATE_IP]:5080 succeeds, else 503.\n- Relevant proofs: 405, 406, 407.\n\n## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_FIXED_20260701\n- Moscow Router Homepage card target after fixed apply: href http://[PRIVATE_IP]:5080, siteMonitor http://[PRIVATE_IP]:18083/router-moscow.\n- Fixed updater handles Homepage service-name-key YAML shape and property-style YAML shape.\n- Relevant proofs: 405, 406, 407, 408.\n\n## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_APPLIED_EXACT_20260701\n- Moscow Router Homepage card exact active YAML shape was service-key style.\n- Applied href: http://[PRIVATE_IP]:5080.\n- Applied siteMonitor: http://[PRIVATE_IP]:18083/router-moscow.\n- Health endpoint service: homelab-router-moscow-health-http.service on edge-vm.\n- Relevant proofs: 406, 409, 410.\n\n## FORUM_PROD_VM160_REBUILD_BASELINE_20260701\n- Old test VM160 was stopped and destroyed cleanly: proof 419, CONFIG_EXISTS=no, VM160_DISKS_LEFT=0.\n- New VM160 forum-prod was recreated on pve02 with 4 vCPU, 8 GiB RAM, 200G disk, MAC BC:24:11:B6:45:ED, IP [PRIVATE_IP]/24, gateway [PRIVATE_IP].\n- New VM identity confirmed: forum-prod / forum-prod.gram1.ru, Debian 12 bookworm, SSH OK, qemu-agent OK, chrony OK.\n- Swapfile 8G enabled with /swapfile mode 600 and fstab entry: proof 430. Proof 429 is invalid due nested command substitution quoting.\n- SURY PHP repo added; PHP 8.5.7 available and installed with Nginx 1.22.1, MariaDB 10.11.14 and required XenForo modules.\n- Beget-compatible profile applied: memory_limit 256M, post/upload 1024M, max_input_vars 10000, MariaDB utf8mb4/utf8mb4_unicode_ci, innodb_buffer_pool_size 2G.\n- Operator reference copies placed in /root/homelab-operator inside VM160 for local VM command preflight.\n\n## FORUM_PROD_CODEVIPE_FIRST_RECIPE_STRATEGY_20260701\n- Current VM160 is now laboratory state, not final production closure.\n- Bulk import of five forums reached DB/files/nginx/php-fpm ready state, but frontend body stayed empty under PHP 8.5.7.\n- Strategy changed: prove CodeVipe first on controlled runtime, likely PHP 8.3 or PHP 8.2, then destroy VM160 and rebuild all five forums from scratch using the proven recipe.\n- Do not publish Cloudflare/NPMplus routes for these forums until CodeVipe recipe is proven and final rebuild is complete.\n- Proofs: failed/diagnostic chain 451-480; next snapshot proof 481 before further lab work.\n\n## FORUM_PROD_FRESH5_DEPLOYED_OK_20260701\n- VM160 forum-prod on pve02, IP [PRIVATE_IP].\n- Runtime: Debian 12, Nginx, MariaDB 10.11, PHP 8.3-FPM.\n- Five XenForo forums deployed from fresh ZIP backups: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- Webroots: /var/www/forums/{codevipe,gamevipe,hkmods,zakrutim,dsmods}/public.\n- DBs: xf_codevipe, xf_gamevipe, xf_hkmods, xf_zakrutim, xf_dsmods.\n- Final health proof inside VM: /root/evidence/513_FORUM_PROD_FRESH5_FINAL_HEALTH_PROOF.txt.\n- Final result: all five frontend/admin HTTP 200, www redirects 301, internal_data 403, no new XenForo errors.\n- Lesson: fresh file ZIPs contain populated internal_data/code_cache; broken branch lacked compiled template cache.\n\n## OPERATOR_RULE_CONTINUE_WITH_NEXT_COMMAND_20260701\n- Rule: when an infrastructure step ends at an expected prompt or safe checkpoint, do not stop with only a status summary.\n- Rule: immediately provide the next executable command in the same response.\n- Applies to: SSH enter/exit points, VM prompt confirmations, snapshot confirmations, file copy confirmations, successful health checks, and other obvious next-step transitions.\n- Exception: stop only if the previous output contains an error, ambiguity, dangerous state, missing file, secret exposure risk, or a required user decision.\n\n## OPERATOR_RULE_BATCH_2_3_COMMANDS_20260701\n- Rule: when safe, provide 2-3 sequential executable commands in one response instead of stopping after each expected checkpoint.\n- Applies after: successful prompt transitions, clean health checks, completed copies, completed snapshots, and non-destructive inventory.\n- Do not batch commands across: destructive operations, public exposure, DNS/SSL changes, secret handling, unclear errors, missing files, or required user decisions.\n- Each command must still print ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK before work.\n\n## FORUM_PROD_FRESH5_PUBLIC_CUTOVER_OK_20260701\n- Public cutover completed for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- Cloudflare A records for main and www names point to edge public IP 95.84.154.183.\n- Edge NPMplus manual routes terminate TLS with real Let’s Encrypt certificates and proxy to forum-prod VM160 at [PRIVATE_IP]:80.\n- Final public proof: /root/evidence/546_PUBLIC_FORUMS_FINAL_DNS_HTTPS_PROOF.txt on pve01.\n- Expected public state: main domains HTTPS 200 with ssl_verify_result 0; HTTP redirects to HTTPS; www redirects to main domain through forum-prod canonical vhosts.\n\n## EDGE_FORUM_PUBLICATION_BACKUP_20260701\n- Edge NPMplus forum publication backup created after public cutover.\n- Backup location: /opt/npmplus/manual-backups/forum-public-ok-*.tar.gz on edge-vm [PRIVATE_IP].\n- Includes manual proxy_host configs 200-204 and Let’s Encrypt forum certificates under /opt/npmplus/tls/forum-certs.\n\n## FORUM_CERT_RENEWAL_CONFIGURED_20260701\n- Edge certificate renewal configured on edge-vm [PRIVATE_IP].\n- Token stored at /opt/npmplus/secure/forum_cf_token.env mode 600 root-only.\n- Renewal script: /opt/npmplus/scripts/renew-forum-certs.sh.\n- Cron: /etc/cron.d/forum-cert-renew, daily 03:17.\n- Proof: /root/evidence/550_EDGE_FORUM_CERT_RENEWAL_SETUP_PROOF.txt on pve01.\n\n## CLOUDFLARE_FORUM_DNS_AUDIT_OK_20260701\n- Full Cloudflare DNS audit completed for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- Proof: /root/evidence/554_CLOUDFLARE_FULL_DNS_AUDIT_PROOF.txt on pve01.\n- Result: root and www A records for all five zones point to 95.84.154.183.\n- Result: no root/www AAAA records, no root/www CNAME conflicts, no records with old IP 87.236.18.45.\n- Remaining cdn.* CNAME and TXT/DMARC/DKIM records were observed and not changed.\n\n## FORUMS_FINAL_DNS_AUDIT_SNAPSHOT_OK_20260701\n- Final post-DNS-audit snapshot completed after shortening Proxmox snapshot name.\n- Snapshot name: forum-dns-ok-065203Z\n- Proof: /root/evidence/557_FINAL_SHORT_SNAPSHOT_AFTER_DNS_AUDIT_PROOF.txt on pve01.\n- Current public state: all five main domains resolve to 95.84.154.183 and return HTTPS 200 with valid certificates.\n\n## FORUM_LOCAL_BACKUP_CONFIGURED_20260701\n- Local forum backup configured inside VM160 forum-prod.\n- Script: /root/scripts/forum-backup.sh.\n- Cron: /etc/cron.d/forum-local-backup, daily 02:42.\n- Retention: removes per-run directories older than 14 days; compressed archives remain under /var/backups/forums.\n- Proof: /root/evidence/560_FORUM_BACKUP_SCRIPT_SETUP_PROOF.txt on pve01.\n\n## FORUM_MAIL_NOREPLY_AND_SPF_CLEANUP_20260701\n- XenForo email fields defaultEmailAddress/contactEmailAddress/bounceEmailAddress/emailSenderName set to noreply@pvepro.ru for all five forums.\n- SPF TXT on codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru changed from stale 87.236.18.45 to v=spf1 -all.\n- DKIM, DMARC and cdn.* CNAME records were not changed.\n- Proofs: 564_UPDATE_XENFORO_EMAIL_NOREPLY_PROOF.txt, 565_CF_UPDATE_FORUM_SPF_NOREPLY_PROOF.txt, 566_FORUM_MAIL_NOREPLY_FINAL_AUDIT_PROOF.txt.\n\n## FORUM_CDN_RECORDS_DELETED_20260701\n- Deleted cdn.* CNAME records for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- Reason: old cdn.* records pointed to previous provider/trbcdn and no current public/local forum references were observed.\n- Root and www A records remain on 95.84.154.183.\n- Proofs: /root/evidence/567_CF_DELETE_FORUM_CDN_RECORDS_PROOF.txt and /root/evidence/568_CDN_DELETE_FINAL_AUDIT_REFERENCE_PROOF.txt on pve01.\n\n## FORUM_SPF_DEDUPED_AND_CDN_DELETED_20260701\n- Fixed duplicate SPF state caused by earlier failed cleanup attempt.\n- Each of codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru now has exactly one SPF TXT: v=spf1 -all.\n- Old SPF references to 87.236.18.* are absent.\n- cdn.* CNAME records are absent.\n- Public HTTPS remained healthy for all five forums.\n- Proofs: 569_CF_FIX_DUPLICATE_SPF_RECORDS_PROOF.txt and 570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt.\n\n## FORUM_PUBLIC_DNS_MAIL_CLEAN_FINAL_20260701\n- Public web and mail-related DNS cleanup completed for five forum domains.\n- @ and www A records point to 95.84.154.183.\n- cdn.* CNAME records removed.\n- SPF deduped to exactly one TXT: v=spf1 -all on each forum domain.\n- Old provider IP 87.236.18.* absent from forum domain TXT records.\n- XenForo visible email identity changed to noreply@pvepro.ru on all forums.\n- Proofs: 570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt, 571_FORUM_MAIL_TRANSPORT_READINESS_PROOF.txt, 572_FORUM_PUBLIC_AND_MAIL_DNS_FINAL_SUMMARY_PROOF.txt.\n\n## FORUM_SMTP_TRANSPORT_PAUSED_20260701\n- Forum XenForo email identity is noreply@pvepro.ru, but actual SMTP transport is paused.\n- Reason: failed SMTP authentication triggered Mailcow netfilter ban for 95.84.154.183.\n- Broken msmtp secret/config files were removed from forum-prod.\n- Do not re-enable SMTP until the mailbox/app password is rotated and tested with a single controlled attempt.\n- Proof: /root/evidence/590_FORUM_SMTP_FAILED_CONFIG_DISABLED_PROOF.txt.\n\n## POST_INCIDENT_STABLE_STATE_20260701\n- Incident after failed forum SMTP testing resolved.\n- Mailcow and NetBird are reachable again: mail.pvepro.ru/admin and nb.pvepro.ru return HTTP 200.\n- Mailcow netfilter ban for 95.84.154.183 was removed on VPS mail.pvepro.ru.\n- Forum sites remain healthy over HTTPS.\n- Forum SMTP transport remains paused; broken msmtp configs/secrets removed from forum-prod.\n- Do not retry SMTP until the mailbox/app password is rotated.\n- Proofs: 589_POST_UNBAN_MAILCOW_NETBIRD_VERIFY.txt on VPS, 590_FORUM_SMTP_FAILED_CONFIG_DISABLED_PROOF.txt, 591_POST_INCIDENT_PUBLIC_STATUS_PROOF.txt, 592_POST_INCIDENT_FINAL_STABLE_STATE_PROOF.txt.\n\n## FORUM_SMTP_ONE_SHOT_TEST_OK_20260701\n- One controlled SMTP test from forum-prod to mail.pvepro.ru:587 succeeded using rotated credentials.\n- No persistent forum SMTP config was enabled.\n- One-shot secret/config files were removed after the test.\n- Mailcow and NetBird remained reachable after the test.\n- Proofs: /root/evidence/594_FORUM_SMTP_ONE_SHOT_TEST_PROOF.txt and /root/evidence/595_SMTP_TEST_AFTERCHECK_PROOF.txt.\n\n## SMTP_ONESHOT_OK_STABLE_FINAL_20260701\n- Rotated SMTP credential was tested once from forum-prod to mail.pvepro.ru:587 successfully.\n- No persistent SMTP config or secret was left on forum-prod after the one-shot test.\n- Mailcow and NetBird remained reachable after the test.\n- Forum websites remained healthy.\n- Persistent forum SMTP is still not enabled; enable it only with an explicit command and hidden password input.\n- Proofs: 594_FORUM_SMTP_ONE_SHOT_TEST_PROOF.txt, 595_SMTP_TEST_AFTERCHECK_PROOF.txt, 596_SMTP_ONESHOT_OK_FINAL_STABLE_PROOF.txt.\n\n## FORUM_PERSISTENT_MSMTP_ENABLED_20260701\n- Persistent msmtp-mta transport enabled on forum-prod as /usr/sbin/sendmail.\n- SMTP relay: mail.pvepro.ru:587 STARTTLS, sender noreply@pvepro.ru.\n- Secret stored only as root:www-data 0640 base64 file under /etc/msmtp/pvepro.env; no raw SMTP_PASS file remains.\n- PHP mail() as www-data succeeded after persistent config installation.\n- Mailcow and NetBird remained reachable after enabling persistent forum SMTP.\n- Proofs: /root/evidence/597_ENABLE_FORUM_PERSISTENT_MSMTP_PROOF.txt and /root/evidence/598_PERSISTENT_MSMTP_FINAL_VERIFY_PROOF.txt.\n\n## FORUM_PERSISTENT_SMTP_FINAL_STABLE_20260701\n- Persistent forum SMTP via msmtp is enabled on forum-prod.\n- PHP mail() as www-data succeeded after installation.\n- Forum domains remain healthy over HTTPS.\n- Mailcow and NetBird remain reachable after enabling persistent forum SMTP.\n- Secret storage is root:www-data 0640 under /etc/msmtp/pvepro.env; raw [SENSITIVE_PATH] SMTP_PASS file is absent.\n- Snapshot after enable: forum-smtp-on-080840Z.\n- Proofs: 597_ENABLE_FORUM_PERSISTENT_MSMTP_PROOF.txt, 598_PERSISTENT_MSMTP_FINAL_VERIFY_PROOF.txt, 600_FORUM_PERSISTENT_SMTP_FINAL_STABLE_PROOF.txt.\n\n## FORUM_PROD_FRESH5_MAILRU_CLOUD_BACKUP_OK_20260701\n- Existing Mail.ru Cloud rclone crypt remotes on pve02 were reused; no new cloud account was configured.\n- Old codevipe-cloud-backup.timer was stale after fresh five-forum rebuild and is disabled.\n- New pve02 timer forum-prod-fresh5-cloud-backup.timer uploads current forum-prod local archives from VM160 /var/backups/forums.\n- Current backup contains all five forums: codevipe, gamevipe, hkmods, zakrutim, dsmods.\n- Upload target: pve02-mail-01/02/03/04-crypt under app/forums/forum-prod/fresh5/.\n- Archive is split into 1500M parts before upload, with SHA256SUMS and MANIFEST.txt.\n- First fresh5 cloud upload proof: /root/evidence/608_FORUM_PROD_FRESH5_CLOUD_BACKUP_FIRST_RUN_PROOF.txt.\n\n## FORUM_PROD_FRESH5_MAILRU_CLOUD_RESTORE_DRILL_OK_20260701\n- Restore drill from Mail.ru Cloud crypt remote completed for forum-prod fresh5 backup.\n- Remote source used: pve02-mail-01-crypt app/forums/forum-prod/fresh5 latest timestamp.\n- Downloaded split parts, MANIFEST.txt, SHA256SUMS and SHA256SUMS.local were verified.\n- Part SHA256 verification passed.\n- Split archive was reconstructed and full archive hash matched SHA256SUMS.local.\n- tar listing confirmed DB dumps and file archives for codevipe, gamevipe, hkmods, zakrutim and dsmods.\n- Proof: /root/evidence/612_FORUM_FRESH5_CLOUD_RESTORE_DRILL_CORRECTED_PROOF.txt.\n\n## FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_OK_20260701\n- Five public XenForo forums are healthy over HTTPS.\n- Mailcow admin and NetBird UI are reachable after persistent forum SMTP enablement.\n- Persistent forum SMTP via msmtp is enabled and tested.\n- Local backup exists on VM160 under /var/backups/forums.\n- Mail.ru Cloud fresh5 backup is enabled on pve02 via forum-prod-fresh5-cloud-backup.timer.\n- Old CodeVipe-only cloud timer is disabled.\n- Cloud restore drill passed: split parts verified, archive reconstructed, SHA256 matched, tar contained DB/file archives for all five forums.\n- Final rollup proof: /root/evidence/614_FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_PROOF.txt.\n\n## FORUM_XENFORO_JOB_RUNNER_CONFIGURED_20260701\n- XenForo CLI job runner configured inside VM160 forum-prod.\n- Script: /root/scripts/forum-xenforo-run-jobs.sh.\n- Cron: /etc/cron.d/forum-xenforo-run-jobs every 5 minutes.\n- Purpose: process XenForo job queues and cron tasks independent of forum traffic.\n- First run proof: /root/evidence/617_XENFORO_RUN_JOBS_ONCE_PROOF.txt.\n- Cron verification proof: /root/evidence/618_XENFORO_JOB_RUNNER_CRON_CONFIG_PROOF.txt.\n\n## FORUM_XENFORO_CRON_DAEMON_AND_JOBRUNNER_OK_20260701\n- cron daemon inside VM160 forum-prod is active/enabled and was proven by temporary /etc/cron.d execution proof.\n- XenForo job runner script executed successfully after cron daemon verification.\n- Due XenForo cron count returned to zero after run.\n- Proof: /root/evidence/621_FORUM_CRON_DAEMON_JOBRUNNER_VERIFY_PROOF.txt.\n\n## FORUM_DBTECH_TOR_TIMEOUT_CLASSIFIED_20260701\n- Each forum had one historical XenForo error_log row for cURL timeout to check.torproject.org/torbulkexitlist.\n- Source addon: DBTech/Security DragonByte Security 5.0.0.\n- dbtech_security_tornodes option is 0 on all five forums.\n- Classified as non-blocking historical external network timeout/noise; not a forum runtime failure.\n- Error rows were not deleted.\n- Proof: /root/evidence/622_TOR_TIMEOUT_CLASSIFICATION_FINAL_PROOF.txt.\n\n## FORUM_XENFORO_BUILTIN_MAIL_DELIVERED_BUT_SPAM_20260701\n- Built-in XenForo outgoing email test from site \"Моды для Hollow Knight\" was delivered to aleisaev@yandex.ru.\n- Sender was noreply@pvepro.ru.\n- Yandex placed the message in Spam with warning that links/images were disabled.\n- This proves forum SMTP transport works, but deliverability/reputation needs tuning.\n- Custom proof 623 is invalid due /root path permission issue and is superseded by this user-observed built-in test.\n- Next evidence needed: Yandex raw message headers, especially Authentication-Results and DKIM-Signature.\n- Proof: /root/evidence/627_XENFORO_BUILTIN_MAIL_DELIVERED_SPAM_CLASSIFICATION_PROOF.txt.\n\n## FORUM_YANDEX_DELIVERABILITY_SPAM_WITH_SPF_DKIM_PASS_20260701\n- Built-in XenForo outgoing email test was delivered to aleisaev@yandex.ru but placed in Spam.\n- User-provided Yandex headers showed SPF pass and DKIM pass for pvepro.ru.\n- Return-Path, From and DKIM domain aligned on pvepro.ru.\n- Yandex spam score observed: X-Yandex-Spam: 4.\n- Classification: SMTP/Mailcow/XenForo transport is OK; remaining issue is deliverability/reputation/content filtering.\n- DNS change is not required based on this header proof.\n- Next actions: mark as Not spam in Yandex, add/check pvepro.ru in Yandex Postmaster, warm up sender reputation, optionally plan per-forum sender domains later.\n\n## FORUM_PROJECT_FINAL_ACCEPTANCE_OK_20260701\n- Five XenForo forums are published and healthy over HTTPS: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- VM160 forum-prod is the active production forum VM on pve02 at [PRIVATE_IP].\n- Runtime services are configured: nginx, MariaDB, PHP 8.3 FPM, qemu-agent, cron, msmtp/sendmail transport.\n- NPMplus on edge terminates TLS and proxies all five forums to VM160.\n- Let’s Encrypt certificates are active for all five domains.\n- Forum SMTP transport works through noreply@pvepro.ru; Yandex headers proved SPF pass and DKIM pass. Spam placement is classified as reputation/content deliverability, not server failure.\n- XenForo job runner cron is configured and cron daemon execution was proven.\n- Local backup is configured on VM160 and Mail.ru Cloud fresh5/all-forums backup is configured on pve02.\n- Cloud restore drill passed: split parts verified, archive SHA matched, tar contained DB/file/config archives for all five forums.\n- DBTech Tor timeout rows are classified as historical external timeout noise and were not deleted.\n- Final proof: /root/evidence/629_FORUM_PROJECT_FINAL_ACCEPTANCE_PROOF.txt.\n\n## FORUM_NEXT_CHAT_HANDOFF_FULL_STATE_20260701\n\n### Start point\n- Start shell: root@pve01.\n- Canonical truth files:\n - /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\n - /etc/pve/31_HOMELAB_REFERENCE.md\n- Before any infra command, strictly check both files and relevant context blocks.\n- Every infra command must print:\n - ERROR_REGISTER_CHECK=OK\n - REFERENCE_CHECK=OK\n\n### Production forum VM\n- VMID: 160.\n- Name: forum-prod.\n- Node: pve02.\n- IP: [PRIVATE_IP].\n- OS: Debian 12.\n- Runtime: nginx, MariaDB 10.11, PHP 8.3 FPM, qemu-agent, cron, msmtp/sendmail transport.\n- Final accepted snapshot: forum-final-accepted-091934Z.\n- Mail final snapshot: forum-mail-ok-091815Z.\n- Postlaunch snapshot: forum-postlaunch-ok-085501Z.\n\n### Public forums\n- codevipe.ru -> Форум CodeVipe.\n- gamevipe.ru -> Форум GameVipe.\n- hkmods.ru -> Моды для Hollow Knight.\n- zakrutim.ru -> Консервирование и закрутки.\n- dsmods.ru -> Секреты и моды Doom.\n- All five are public HTTPS 200 with valid TLS.\n- Root/www DNS points through edge public IP 95.84.154.183.\n- NPMplus on edge terminates TLS and proxies to http://[PRIVATE_IP]:80.\n\n### Mail\n- Forum sender: noreply@pvepro.ru.\n- Transport: XenForo sendmail -> msmtp -> mail.pvepro.ru.\n- Built-in XenForo mail test reached Yandex.\n- Yandex headers showed SPF pass and DKIM pass for pvepro.ru.\n- Yandex placed the test in Spam with X-Yandex-Spam: 4.\n- Classification: transport/authentication OK; remaining issue is sender reputation/content filtering.\n- DNS change is not required from the captured header proof.\n- Manual next action: click “Не спам!” in Yandex and add/check pvepro.ru in Yandex Postmaster.\n\n### Backups and restore\n- Local backup configured inside VM160.\n- Local archive path pattern: /var/backups/forums/forum-backup-*.tar.gz.\n- pve02 Mail.ru Cloud fresh5/all-forums backup configured.\n- New timer: forum-prod-fresh5-cloud-backup.timer active/enabled.\n- Old codevipe-only timer disabled/inactive.\n- Cloud remotes:\n - pve02-mail-01-crypt\n - pve02-mail-02-crypt\n - pve02-mail-03-crypt\n - pve02-mail-04-crypt\n- Latest verified cloud stage during final acceptance: 20260701T083354Z.\n- Restore drill passed: split parts verified, archive SHA matched, tar contained DB/file/config archives for all five forums.\n\n### XenForo jobs and cron\n- cron daemon was missing, then installed and enabled.\n- /etc/cron.d execution was proven by temporary cron proof.\n- XenForo job runner configured:\n - /root/scripts/forum-xenforo-run-jobs.sh\n - /etc/cron.d/forum-xenforo-run-jobs\n- Purpose: process XenForo jobs and cron tasks independent of visitor traffic.\n\n### DBTech/Tor timeout\n- Each forum had one historical xf_error_log row for cURL timeout to check.torproject.org.\n- Source addon: DBTech/Security DragonByte Security 5.0.0.\n- dbtech_security_tornodes=0 on all five forums.\n- Classified as non-blocking historical external timeout noise.\n- Rows were not deleted.\n\n### Key final proofs\n- /root/evidence/629_FORUM_PROJECT_FINAL_ACCEPTANCE_PROOF.txt\n- /root/evidence/628_FORUM_MAIL_DELIVERABILITY_FINAL_SNAPSHOT_PROOF.txt\n- /root/evidence/627_YANDEX_HEADER_DELIVERABILITY_CLASSIFICATION_PROOF.txt\n- /root/evidence/622_TOR_TIMEOUT_CLASSIFICATION_FINAL_PROOF.txt\n- /root/evidence/621_FORUM_CRON_DAEMON_JOBRUNNER_VERIFY_PROOF.txt\n- /root/evidence/614_FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_PROOF.txt\n- /root/evidence/613_FORUM_FRESH5_CLOUD_RESTORE_DRILL_FINALIZE_PROOF.txt\n- /root/evidence/612_FORUM_FRESH5_CLOUD_RESTORE_DRILL_CORRECTED_PROOF.txt\n- /root/evidence/609_FORUM_PROD_FRESH5_CLOUD_BACKUP_FINALIZE_PROOF.txt\n- /root/evidence/608_FORUM_PROD_FRESH5_CLOUD_BACKUP_FIRST_RUN_PROOF.txt\n- /root/evidence/600_FORUM_PERSISTENT_SMTP_FINAL_STABLE_PROOF.txt\n- /root/evidence/570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt\n- /root/evidence/560_FORUM_BACKUP_SCRIPT_SETUP_PROOF.txt\n\n## PARKED_DOMAINS_PLACEHOLDER_PUBLIC_OK_20260701\n\n- Parked/placeholder public page is live for three unused domains: newfi.ru, hapusya.ru and kingofwolk.ru.\n- Public DNS root and www hostnames already point to edge public IP 95.84.154.183.\n- Local placeholder service on edge-vm: homelab-parked-domains-http.service, enabled/active, serves marker PARKED_PAGE_OK on http://127.0.0.1:18088.\n- NPMplus manual managed route files: /data/nginx/proxy_host/998.conf for newfi.ru, 997.conf for hapusya.ru and 996.conf for kingofwolk.ru.\n- Public HTTPS validation from pve01 after final route-only finalize returned HTTP 200 and PARKED_PAGE_OK for all six hostnames: root and www for all three domains.\n- Certificates were issued by certbot DNS-01 using Cloudflare API hooks; HTTP-01 must not be retried for these parked domains without a new plan because earlier attempts hit NPMplus default redirect/include-order behavior.\n- Cloudflare token value is not stored in the reference; token file path only: /opt/npmplus/secure/parked_cf_token, root-owned mode 600.\n- Certificate names on edge-vm: parked-newfi.ru, parked-hapusya.ru and parked-kingofwolk.ru; observed expiry during setup: 2026-09-29.\n- Certbot deploy hook installed: /etc/letsencrypt/renewal-hooks/deploy/parked-domains-npmplus-deploy.sh.\n- Health file: /var/lib/homelab-health/parked-domains-public.txt.\n- Final route backup: /opt/npmplus/manual-backups/parked-stage16-route-only-20260701T155843Z.\n- Final proof: /root/evidence/645_PARKED_DOMAINS_STAGE16_ROUTE_ONLY_FINALIZE_PROOF.txt.\n\n## GRAM1_ROOT_WWW_PLACEHOLDER_PUBLIC_OK_20260701\n\n- gram1.ru root and www.gram1.ru are routed to the existing parked placeholder page on edge-vm.\n- Existing gram1.ru subdomain routes were not changed.\n- NPMplus managed route file: /data/nginx/proxy_host/995.conf.\n- Upstream placeholder: http://127.0.0.1:18088.\n- Certificate name on edge-vm: parked-gram1.ru.\n- Certificate issuance mode: certbot DNS-01 with dedicated gram1 Cloudflare token file /opt/npmplus/secure/gram1_cf_token.\n- Token value is not stored in reference; token file is root-owned mode 600.\n- Certbot deploy hook: /etc/letsencrypt/renewal-hooks/deploy/gram1-root-npmplus-deploy.sh.\n- Health file: /var/lib/homelab-health/gram1-root-placeholder.txt.\n- Closure proof: /root/evidence/653_GRAM1_ROOT_WWW_PLACEHOLDER_CLOSE_STAGE19_PROOF.txt.\n\n## PVEPRO_ROOT_WWW_EDGE_LANDING_PUBLIC_OK_20260701\n\n- pvepro.ru and www.pvepro.ru root/www were moved from the Mailcow VPS default web surface to an edge landing page.\n- mail.pvepro.ru and nb.pvepro.ru were intentionally not changed.\n- Cloudflare A records for pvepro.ru and www.pvepro.ru point to edge public IP 95.84.154.183.\n- NPMplus managed route file: /data/nginx/proxy_host/994.conf.\n- Landing service on edge-vm: homelab-pvepro-landing-http.service on http://127.0.0.1:18089.\n- Landing marker: PVEPRO_LANDING_OK.\n- Certificate name on edge-vm: landing-pvepro.ru.\n- Certificate issuance mode: certbot DNS-01 with dedicated pvepro Cloudflare token file /opt/npmplus/secure/pvepro_cf_token.\n- Token value is not stored in reference; token file is root-owned mode 600.\n- Certbot deploy hook: /etc/letsencrypt/renewal-hooks/deploy/pvepro-root-npmplus-deploy.sh.\n- Health file: /var/lib/homelab-health/pvepro-root-landing.txt.\n- Closure proof: /root/evidence/658_PVEPRO_STAGE23_FINAL_READONLY_CLOSE_PROOF.txt.\n\n## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701\n\n- taftauto.ru is the dacha router domain.\n- Current public A record observed during audit: 194.33.48.131.\n- Router model family: Netcraze-like, same as Moscow router family by operator statement.\n- Safe private management access is not available yet.\n- Do not expose the router admin interface publicly for certificate automation.\n- Certificate auto-renewal/deploy to the router is intentionally blocked until a private access path exists.\n- Recommended future path: establish VPN/NetBird/WireGuard or reverse-proxy/private management endpoint first; then issue DNS-01 certificate on a trusted node and deploy cert/key to the router only over that private path.\n- Closure status: documented blocker, not runtime outage.\n- Read-only audit proof: /root/evidence/654_PVEPRO_TAFTAUTO_EXTERNAL_READONLY_AUDIT_PROOF.txt.\n\n## DOMAIN_PORTFOLIO_FINAL_STATUS_20260701\n\n- Public parked placeholder domains closed: newfi.ru, hapusya.ru, kingofwolk.ru.\n- gram1.ru root and www.gram1.ru closed on the same placeholder page; existing gram1.ru service subdomains were not changed.\n- pvepro.ru root and www.pvepro.ru closed on a separate edge landing page; mail.pvepro.ru and nb.pvepro.ru remain on the external VPS/IPs and were validated after the change.\n- Forum domains remain XenForo on forum-prod through edge NPMplus: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- taftauto.ru is documented as blocked for certificate autodeploy until private router management access exists.\n- Forum renewal one-shot proof from Stage18 returned OK.\n- Parked certbot dry-run passed for newfi.ru and hapusya.ru; kingofwolk.ru dry-run hit transient Let’s Encrypt service-busy/rateLimited after a valid active certificate and working public HTTPS were already confirmed.\n- Current finalization proof: /root/evidence/661_DOMAIN_PORTFOLIO_FINAL_CLOSE_AND_TAFTAUTO_BLOCKED_PROOF.txt.\n\n## TAFTAUTO_WG_PRIVATE_MANAGEMENT_OK_20260702\n- Public SSH 194.33.48.131:22 closed.\n- WireGuard management path OK: edge [PRIVATE_IP] -> dacha router [PRIVATE_IP].\n- Router HTTP over WG returns HTTP/1.1 200 OK / Ndm-Sysmode: router.\n- Proof: /root/evidence/665_TAFTAUTO_WG_PRIVATE_MANAGEMENT_OK_20260702_PROOF.txt\n\n## TAFTAUTO_WG_PSK_ROTATED_AND_MANAGEMENT_OK_20260702\n- WireGuard PSK rotated after leaked self-test.\n- Dacha active interface after reimport: Wireguard1.\n- Public SSH remains closed.\n- Private management OK: edge [PRIVATE_IP] -> dacha [PRIVATE_IP] via WG.\n- Proof: /root/evidence/666_TAFTAUTO_WG_PSK_ROTATED_AND_MANAGEMENT_OK_20260702_PROOF.txt\n\n## TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702\n- Supersedes proof 666 because proof 666 showed PUBLIC_SSH_22_STILL_OPEN.\n- WireGuard PSK rotated.\n- Public SSH closed.\n- Private WG management OK: edge [PRIVATE_IP] -> dacha [PRIVATE_IP].\n- Proof: /root/evidence/667_TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702_PROOF.txt\n\n## TAFTAUTO_CERT_DNS01_ISSUED_20260702\n- DNS-01 certificate issued on edge-vm for taftauto.ru and www.taftauto.ru.\n- Cert name: router-taftauto.ru.\n- Cert path: /etc/letsencrypt/live/router-taftauto.ru/fullchain.pem.\n- Key path: /etc/letsencrypt/live/router-taftauto.ru/privkey.pem.\n- Proof: /root/evidence/668_TAFTAUTO_CERT_DNS01_ISSUED_20260702_PROOF.txt\n\n## TAFTAUTO_PUBLIC_EDGE_WG_ROUTER_FINAL_OK_20260702\n- Supersedes failed/partial proof 669 because direct SSH was open during that run.\n- taftauto.ru and www.taftauto.ru point to edge public IP 95.84.154.183.\n- NPMplus route: /opt/npmplus/nginx/proxy_host/993.conf.\n- TLS terminates on edge using certbot DNS-01 cert router-taftauto.ru.\n- Upstream is private WireGuard path: edge [PRIVATE_IP] -> dacha router [PRIVATE_IP]:80.\n- Public HTTPS returns router page with x-taftauto-router: managed and ndm-sysmode: router.\n- Direct dacha public SSH is closed; WG SSH remains open.\n- Proof: /root/evidence/670_TAFTAUTO_PUBLIC_EDGE_WG_ROUTER_FINAL_OK_20260702_PROOF.txt\n\n## HOMEPAGE_EXTERNAL_SMARTAPE_AND_SITES_OK_20260702\n- Homepage services.yaml repaired with YAML-aware edit after bad indentation issue.\n- SmartApe card added to External Homelabs.\n- Router Moscow card removed.\n- Sites category added with 11 site cards: CodeVipe, GameVipe, HKMods, Zakrutim, DSMods, Newfi, Hapusya, KingOfWolk, Gram1, PVEPro, TaftAuto.\n- Proof: /root/evidence/671_HOMEPAGE_EXTERNAL_SMARTAPE_AND_SITES_OK_20260702_PROOF.txt\n\n## TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK_20260702\n- Certbot renew dry-run for router-taftauto.ru succeeds with manual DNS-01 hooks.\n- Deploy hook installed: /etc/letsencrypt/renewal-hooks/deploy/taftauto-router-npmplus-deploy.sh.\n- Deploy hook copies renewed cert/key into NPMplus /data/tls/router-certs/taftauto.ru and reloads nginx after nginx -t.\n- Proof: /root/evidence/672_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK_20260702_PROOF.txt\n\n## TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_INSTALLED_DRYRUN_RATELIMITED_20260702\n- Supersedes partial proof 672.\n- Deploy hook installed and manual deploy invocation works.\n- NPMplus cert copy and nginx config validate OK.\n- Public taftauto.ru remains OK.\n- Certbot dry-run retry is deferred due Let's Encrypt rateLimited / Service busy.\n- Proof: /root/evidence/673_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_INSTALLED_DRYRUN_RATELIMITED_20260702_PROOF.txt\n\n## VPN_GRAM1_ALIAS_ROUTES_REMOVED_20260702\n- Removed obsolete NPMplus route aliases without Homepage cards: auth.vpn.gram1.ru, gitea.vpn.gram1.ru, homepage.vpn.gram1.ru, kuma.vpn.gram1.ru.\n- NPMplus nginx config validates after removal.\n- Proof: /root/evidence/674_VPN_GRAM1_ALIAS_ROUTES_REMOVED_20260702_PROOF.txt\n\n## HOMELAB_HOMEPAGE_AND_TAFTAUTO_FINAL_OK_20260702\n- Homepage final layout/category state confirmed: YAML validates, categories use row/columns layout, logs show no YAMLException.\n- TaftAuto public HTTPS works for taftauto.ru and www.taftauto.ru through edge/NPMplus.\n- Direct public SSH to dacha router is closed.\n- Private WG SSH to dacha router remains open.\n- NPMplus nginx config validates.\n- Proof: /root/evidence/677_HOMELAB_HOMEPAGE_AND_TAFTAUTO_FINAL_OK_20260702_PROOF.txt\n\n## CLOUDFLARE_TOKEN_AUDIT_20260702\n- Cloudflare token files audited without printing secrets.\n- Token files exist, expected permissions were checked, tokens verify active, and expected zone access was checked.\n- Proof: /root/evidence/678_CLOUDFLARE_TOKEN_AUDIT_20260702_PROOF.txt\n\n## EVIDENCE_REVIEW_INDEX_20260702\n- Read-only evidence index created for review/superseded proof cleanup planning.\n- No evidence files were deleted or modified.\n- Proof: /root/evidence/679_EVIDENCE_REVIEW_INDEX_20260702_PROOF.txt\n\n## EVIDENCE_SUPERSEDED_MAP_20260702\n- Evidence superseded map created. No evidence files were deleted.\n- 666, 669, 672_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK, and 676 are not authoritative for final state.\n- 677 is current final Homepage/TaftAuto authority; 678 is current Cloudflare token audit authority.\n- Proof: /root/evidence/680_EVIDENCE_SUPERSEDED_MAP_20260702_PROOF.txt\n\n## HOMEPAGE_LOCAL_HEALTH_MONITORS_OK_20260702\n- Local edge health endpoint is used for Homepage monitoring of Роутер Москва and NPMplus.\n- Роутер Москва href remains external, while siteMonitor uses http://[PRIVATE_IP]:18091/router-moscow.\n- NPMplus siteMonitor uses http://[PRIVATE_IP]:18091/npmplus.\n- Homepage YAML validates and logs show no YAMLException.\n- Proof: /root/evidence/681_HOMEPAGE_LOCAL_HEALTH_MONITORS_OK_20260702_PROOF.txt\n\n## HOMEPAGE_SELF_CARD_REMOVED_FROM_CORE_20260702\n- Removed Homepage self-referential card from Core.\n- Homepage YAML validates and logs show no YAMLException.\n- Proof: /root/evidence/682_HOMEPAGE_SELF_CARD_REMOVED_FROM_CORE_20260702_PROOF.txt\n\n## CERT_RENEW_HOOKS_READONLY_AUDIT_20260702\n- Read-only audit of cert renewal cron entries, renewal configs, deploy hooks, and related script presence completed.\n- No certbot renew/dry-run was executed.\n- Proof: /root/evidence/683_CERT_RENEW_HOOKS_READONLY_AUDIT_20260702_PROOF.txt\n\n## CERT_RENEW_MAPPING_READONLY_AUDIT_20260702\n- Read-only mapping audit completed for renewal confs, deploy hooks, and forum renewal script.\n- No certbot renew/dry-run was executed and no secrets were printed.\n- Proof: /root/evidence/684_CERT_RENEW_MAPPING_READONLY_AUDIT_20260702_PROOF.txt\n\n## CERT_DEPLOY_HOOK_LINEAGE_GUARDS_OK_20260702\n- Deploy hooks hardened with RENEWED_LINEAGE guards for gram1, parked domains, and pvepro.\n- TaftAuto hook already had lineage guard and remains guarded.\n- bash -n validates all checked deploy hooks.\n- No certbot renew/dry-run was executed.\n- Proof: /root/evidence/685_CERT_DEPLOY_HOOK_LINEAGE_GUARDS_OK_20260702_PROOF.txt\n\n## PUBLIC_HTTPS_AND_CERT_EXPIRY_READONLY_20260702\n- Read-only public HTTPS and certificate expiry snapshot completed for forum, parked, gram1, pvepro, and taftauto domains.\n- No certbot renew/dry-run was executed.\n- Proof: /root/evidence/686_PUBLIC_HTTPS_AND_CERT_EXPIRY_READONLY_20260702_PROOF.txt\n\n## NPMPLUS_ROUTE_CERT_MAPPING_READONLY_20260702\n- Read-only NPMplus route to certificate mapping audit completed.\n- Expected proxy routes were found and expected certificate files are present and valid for more than 30 days.\n- No certbot renew/dry-run was executed.\n- Proof: /root/evidence/687_NPMPLUS_ROUTE_CERT_MAPPING_READONLY_20260702_PROOF.txt\n\n## TODAY_PROOFS_SECRET_SCAN_READONLY_20260702\n- Read-only filename-only secret pattern scan completed for today proof files and active reference/error register.\n- No matching secret value patterns were found.\n- Proof: /root/evidence/688_TODAY_PROOFS_SECRET_SCAN_READONLY_20260702_PROOF.txt\n\n## HOMELAB_20260702_SESSION_CLOSURE_OK\n- Session closure proof created for final OK chain 677-688.\n- TaftAuto, Homepage, Cloudflare tokens, evidence map, cert hooks, deploy guards, public HTTPS, route/cert mapping, and secret scan are documented.\n- Proof: /root/evidence/689_HOMELAB_20260702_SESSION_CLOSURE_OK_PROOF.txt\n\n## CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702\n- edge-vm CrowdSec CAPI is registered and enabled through NetBird exit route.\n- Client peer: edge-vm.netbird.selfhosted / 100.100.60.182.\n- Primary exit routing peer: relay.netbird.selfhosted / 100.100.19.1 / Moldova.\n- Backup egress peer present: mail.netbird.selfhosted / 100.100.147.204 / USA.\n- NetBird exit route proof on edge-vm: wg allowed-ips includes 0.0.0.0/0 via relay; ip route get 1.1.1.1 uses wt0 table 7120.\n- External trace after route: 85.121.4.192 / OTP, not home 95.84.154.183 / ARN.\n- CrowdSec proof: cscli lapi status OK, cscli capi status OK, CAPI sharing/blocklist pull enabled, no DISABLE_ONLINE_API or -no-capi flags in active compose.\n- WARP is intentionally absent and must not be reintroduced for this path.\n- Unauthenticated https://api.crowdsec.net/v3/watchers/login returning HTTP 403 is expected network reachability proof.\n- Final proof: /root/evidence/771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_20260702T191057Z_PROOF.txt\n\n## HOMEPAGE_RU_LAYOUT_MONITORING_PROPOSALS_20260702\n- Homepage live config is on core-apps VM [PRIVATE_IP] under /opt/stacks/homepage/config.\n- Groups/descriptions were normalized to Russian.\n- Layout is row/6-columns for all active groups.\n- siteMonitor fields are preserved; monitors must be repaired, not removed.\n- Cloudflare card must be left untouched by explicit operator request.\n- Red badges after migration were primarily caused by Homepage container resolving *.vpn.gram1.ru to edge NetBird IP and timing out; fixed by mapping vpn hostnames to edge LAN [PRIVATE_IP] in Homepage compose extra_hosts.\n- Router Moscow monitor uses edge health endpoint http://[PRIVATE_IP]:18091/router-moscow.\n- NPMplus monitor uses edge health endpoint http://[PRIVATE_IP]:18091/npmplus.\n- Missing-card proposal report: /var/lib/homelab-health/homepage-missing-card-proposals-final.txt.\n\n## UPTIME_KUMA_MISSING_MONITOR_AND_PROPOSALS_20260702\n- Uptime Kuma lives on monitoring VM [PRIVATE_IP].\n- Added one missing monitor: NPMplus Edge Health -> http://[PRIVATE_IP]:18091/npmplus.\n- Uptime Kuma DB backup was created before DB mutation under /root/uptime-kuma-manual-backups/773a-* on monitoring VM.\n- DB integrity after insertion was OK.\n- Deep monitoring proposal report: /var/lib/homelab-health/uptime-kuma-monitoring-proposals.txt.\n- Cloudflare must not be touched by operator request.\n- Recommended next monitors are P0 only after operator approval: Router Moscow Health, Backup/Restore Dashboard, Restore Drill Index, NetBird Peers, CrowdSec CAPI, Paperless Restore.\n\n## UPTIME_KUMA_NPMPLUS_ADMIN_REPAIR_20260702\n- Existing Uptime Kuma monitor \"NPMplus Admin\" was red because it checked https://[PRIVATE_IP]:81/.\n- NPMplus admin is intentionally localhost-bound on edge-vm, so monitoring VM should not check the admin UI directly.\n- Fixed monitor target to edge health endpoint: http://[PRIVATE_IP]:18091/npmplus.\n- Monitor was repaired, not deleted.\n- Cloudflare was not touched.\n\n## UPTIME_KUMA_APPROVED_MONITOR_BATCH_20260702\n- Approved Uptime Kuma monitor batch installed after Homepage/Kuma analysis.\n- Existing NPMplus Admin monitor was repaired to http://[PRIVATE_IP]:18091/npmplus, not deleted.\n- Edge health wrapper /usr/local/sbin/router-moscow-health provides safe endpoints on [PRIVATE_IP]:18091 for router-moscow, npmplus, loki, alloy, cadvisor, registry-cache, socket-proxy, diun, kopia and crowdsec.\n- Cloudflare was not touched by explicit operator request.\n- Uptime Kuma DB backups are under /root/uptime-kuma-manual-backups/773d-approved-batch-* on monitoring VM before the DB mutation.\n\n## UPTIME_KUMA_PENDING_HEALTH_MONITORS_FIX_20260702\n- Four pending Uptime Kuma keyword monitors were repaired by using local pve01 health endpoint paths instead of public backup.gram1.ru paths:\n - Backup Restore Dashboard -> http://[PRIVATE_IP]:9101/backup-restore-dashboard.txt\n - Restore Drill Index -> http://[PRIVATE_IP]:9101/restore-drill-index.txt\n - Paperless Restore -> http://[PRIVATE_IP]:9101/paperless-restore.txt\n - AdGuard Rewrite Sync -> http://[PRIVATE_IP]:9101/adguard-rewrite-sync.txt\n- Cloudflare was not touched.\n- Monitors were repaired, not deleted.\n\n## UPTIME_KUMA_FOUR_HEALTH_SOURCES_REPAIRED_20260702\n- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR.\n- Repaired pve01 health source files under /var/lib/homelab-health:\n - backup-restore-dashboard.txt\n - restore-drill-index.txt\n - paperless-restore.txt\n - adguard-rewrite-sync.txt\n- Kuma monitors point to local pve01 health endpoints on http://[PRIVATE_IP]:9101/.\n- Cloudflare was not touched.\n- Monitors were repaired, not deleted.\n\n## DOCKGE_DISCONNECTED_HOSTS_AND_STALE_STACKS_20260702\n- Dockge is currently present only on edge-vm.\n- core-apps and monitoring run Docker containers but have no Dockge/Dockge-agent detected.\n- Dockge \"2 not connected\" should be interpreted as two disconnected Docker hosts unless later evidence shows otherwise.\n- No stopped containers were found across edge-vm, core-apps and monitoring during analysis.\n- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring.\n- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification.\n\n## DOCKGE_REMOTE_AGENTS_INSTALLED_20260702\n- Dockge main instance runs on edge-vm [PRIVATE_IP].\n- Remote Dockge agents were installed on:\n - core-apps [PRIVATE_IP]:5001\n - monitoring [PRIVATE_IP]:5001\n- Edge Dockge agent table stores the two agent URLs with username/password; password must never be printed.\n- No runtime containers were deleted.\n- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first.\n\n## DOCKGE_REMOTE_AGENTS_AND_STALE_ARCHIVE_FINAL_20260702\n- Dockge main instance: edge-vm [PRIVATE_IP].\n- Remote agents connected: core-apps [PRIVATE_IP]:5001, monitoring [PRIVATE_IP]:5001.\n- Stale moved edge stack folders archived under /opt/dockge-stale-archive/20260702T230442Z on edge-vm; no runtime containers were deleted.\n- External remote stacks exposed to Dockge using bind mounts: /opt/vaultwarden-compose, /opt/gotify-compose, /opt/uptime-kuma-compose into /opt/stacks.\n- Final target: remote not_visible count must be 0 and edge stale remaining matches must be 0.\n\n## DOCKGE_FINAL_MANAGED_STACKS_20260702\n- Dockge main stack, npmplus and remote dockge-agent stacks were recreated from /opt/stacks so Dockge can manage them.\n- Edge Dockge listens on 127.0.0.1:5001; LAN probe to [PRIVATE_IP]:5001 may return 000 and is not the correct health proof.\n- Remote agents listen on [PRIVATE_IP]:5001 and [PRIVATE_IP]:5001.\n- Stale moved stacks were archived, not deleted.\n\n## SERVICE_SETUP_ROADMAP_DEEP_20260702\n- Configuration order: secrets, DNS/proxy/TLS, backup/restore, monitoring, SSO, docs, core data apps, sensitive apps, productivity, media, automation, utilities.\n- Do not configure data-heavy or automation services before backup and monitoring are proven.\n- Per-service done criteria: URL, TLS, admin, SSO decision, volumes, backup, restore test, Kuma, Homepage, docs.\n\n## PHASE00_FREEZE_CLOSED_20260702\n- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain.\n- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml.\n- Old systemd unit homelab-registry-cache.service was disabled and masked because it recreated the container without Compose labels.\n- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/.\n- Next phase: Vaultwarden and break-glass secret baseline.\n\n## PHASE00_FREEZE_CLOSED_20260702\n- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain.\n- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml.\n- Old systemd unit homelab-registry-cache.service was archived and masked because it recreated the container without Compose labels.\n- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/.\n- Next phase: Vaultwarden and break-glass secret baseline.\n\n## PHASE00_FREEZE_CLOSED_20260702\n- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain.\n- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml.\n- Old systemd unit homelab-registry-cache.service was archived and masked with /dev/null because it recreated the container without Compose labels.\n- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/.\n- Next phase: Vaultwarden and break-glass secret baseline.\n\n## OPERATOR_RULE_CLOSE_TAILS_IMMEDIATELY_20260714\n- Жёсткое правило: хвосты не оставлять. Любая проблема, временный артефакт, неудачная проверка, блокер или анти-регрессия, обнаруженные в текущем scope, должны быть устранены и закрыты до перехода к следующей задаче.\n- После любой неуспешной команды текущая задача остаётся активной до установления точной причины, исправления, проверки исправления, добавления анти-регрессии, очистки временных артефактов и выпуска закрывающего proof.\n- Этап разрешено помечать CLOSED только при UNRESOLVED_TAIL_COUNT=0, BLOCKER_COUNT=0, TEMPORARY_ARTIFACT_COUNT=0, проверенном rollback, зелёном health, обновлённых proof и reference.\n- Запрещено откладывать выявленный хвост только ради удобства или перехода к следующему этапу.\n- Если технически необходим отдельный подэтап, он должен быть ограничен точным scope, выполнен и закрыт немедленно до возврата к основной работе.\n- Исключения: внешняя зависимость, опасная неоднозначность, риск раскрытия секрета, отсутствующий обязательный файл или явное решение пользователя. В таком случае статус должен быть BLOCKED или OPEN с точным блокером; статус CLOSED запрещён.\n- Следующий этап не начинается, пока текущий хвост не закрыт.\n- Proof: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/report.txt\n- Secret scan: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/strict-secret-scan.txt\n\n## CLUSTER_ADMIN_STAGE4H_SCOPE_DEFINED_20260714\n- Stage4G остаётся закрытым и стабильно работает в режиме observe-notify.\n- Закрыты два хвоста dependency-аудита: глобальный LIMIT всего UNION ALL и узкий шаблон collector_patch_required.\n- Migration plan содержит collector_patch_required=true; dynamic contract содержит collector_patch_required_after_stage4c=true. Это два подтверждения одного обязательного требования.\n- Migration 003 не применена, 12 новых столбцов отсутствуют.\n- Migration выполняет полный UPDATE observations, четыре SET NOT NULL, пять VALIDATE CONSTRAINT и пять неконкурентных CREATE INDEX.\n- Текущий collector не формирует обязательный provenance-набор; готовый collector patch отсутствует.\n- Production adapters и production apply phase отсутствуют.\n- Отдельный apply migration 003 без collector patch запрещён.\n- Безопасный порядок: collector patch candidate → временная БД и acceptance/rollback → контролируемый migration+collector cutover → adapter integration.\n- Текущий этап открыт: 4H_COLLECTOR_PROVENANCE_PATCH_AND_MIGRATION003_PREAPPLY_READINESS.\n- Предыдущие хвосты закрыты; Stage4H не считается CLOSED до реализации и полного seal.\n- Proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt\n- Secret scan: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/strict-secret-scan.txt\n\n## CLUSTER_ADMIN_STAGE4H_COLLECTOR_BASELINE_AUDIT_CLOSED_20260714\n- Исправленный baseline-аудит collector завершён.\n- collector.py: 113 строк, SHA256 подтверждён.\n- Привилегированные файлы нельзя читать через caller-side `< file` перед sudo; применён sudo wc без внешнего перенаправления.\n- Фактические поля collector_runs: finished_at и error_text; completed_at и error отсутствуют.\n- Найдены один canonical source instance и один collector_runs_foundation_enrich_trg.\n- Trigger обогащает collector_runs полями source_instance_id и run_key.\n- Текущий collector ещё не пишет provenance-поля observations; migration 003 не применена.\n- Production осталась 0|0|OK, timer активен, failed units отсутствуют.\n- Baseline-аудит закрыт без хвостов; Stage4H остаётся OPEN для isolated collector provenance patch candidate.\n- Proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt\n- Secret scan: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/strict-secret-scan.txt\n\n## CLUSTER_ADMIN_STAGE4H_COLLECTOR_PATCH_CANDIDATE_CLOSED_20260714\n- Из точного live collector создан isolated provenance patch candidate; production collector не изменён.\n- Candidate пишет все 12 migration003 provenance-полей и применяет SHA256 fallback-дедупликацию.\n- Metadata разрешает только instance, job, mountpoint и device, ограничена по типам и размеру.\n- Candidate скомпилирован локально и проверен на VM180 Python 3.11.2 от clusteradmin.\n- SSH stdin harness исправлен: bash -s получает ровно path, SHA256 и bytes; первый аргумент дополнительно проверяется по безопасному шаблону.\n- Self-test и отрицательный CLI-тест RC64 прошли; временный remote-файл удалён.\n- Production осталась 0|0|OK; live collector и timer не изменены.\n- Неудачная попытка закрыта без хвостов; Stage4H остаётся OPEN до temporary-DB acceptance и controlled cutover.\n- Candidate root: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z\n- Proof: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/report.txt\n- Secret scan: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/strict-secret-scan.txt\n\n## OPERATOR_RULE_NO_OVERSIZED_MONOLITH_COMMANDS_20260714\n- Запрещены чрезмерно длинные интерактивные однострочные команды с глубокой вложенностью SSH, SQL, Python, awk и кавычек.\n- Рекомендуемый предел интерактивной команды: 8000 байт. Превышение допускается только для простого текста без вложенных языков.\n- Сложная операция оформляется как отдельный versioned task-скрипт с contract, syntax-check, dry-run, manifest, rollback и proof.\n- Создание task-скрипта и его запуск выполняются разными короткими командами.\n- Remote stdout/stderr всегда сохраняются до проверки RC; запрещено терять вывод из-за errexit-sensitive command substitution.\n- Перед запуском проверяются SHA256, размер, владелец, режим и синтаксис task-скрипта.\n- Обрезанная, повреждённая или частично вставленная команда считается невыполненной; сначала проводится side-effect audit.\n- Хвост parse-ошибки закрывается только после доказательства отсутствия изменений и временных артефактов.\n- Proof: /var/lib/homelab-change-sets/operator-command-size-guard-v1/20260714T182653Z/report.txt\n\n## HOMELAB_ADMIN_CLI_CONTRACT_20260714\n- Поддерживаемые команды: version, status, lint TASK_DIR, validate TASK_DIR, run TASK_DIR PHASE.\n- Отдельная команда --help отсутствует. Неизвестный аргумент печатает usage и штатно возвращает RC=64.\n- Нельзя трактовать RC=64 от --help как неисправность runner.\n- Для проверки доступности runner использовать homelab-admin version; для структуры CLI использовать зафиксированную usage-строку.\n- Proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v1/20260714T183338Z/report.txt\n\nAUTOMATIC_CLUSTER_CONTEXT_REFRESH_20260721\nCollector: /usr/local/sbin/homelab-context-collect\nScheduler: hourly at minute 17 via /etc/cron.d/homelab-context-refresh\nDestination: immutable Gitea runtime history with CONTEXT-AUTO UTC command IDs.\n\nAUTOMATIC_CLUSTER_CONTEXT_REFRESH_DIRECT_20260721\nPublisher: /usr/local/sbin/homelab-context-refresh-direct\nSchedule: hourly at minute 17.\nDestination: immutable assistant-runtime history with CONTEXT-AUTO UTC IDs.\nDETAIL=SECTION_REFERENCE_REGISTER_END\nDETAIL=SECTION_ERROR_REGISTER_BEGIN\n# HOMELAB ASSISTANT ERROR REGISTER\n\nНазначение: перед каждой следующей командой сверяться с этим файлом.\n\n## Критические ошибки ассистента\n1. Повторно дал слишком большой интерактивный paste в shell.\n2. Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт.\n3. Дал генератор справочника прямо в терминал вместо безопасного маленького шага.\n4. Нарушил своё же правило: не давать длинные вложенные команды с кавычками.\n\n## Жёсткие правила перед каждой командой\nCHECK-1: команда не должна быть большим paste.\nCHECK-2: команда не должна содержать большой here-doc.\nCHECK-3: команда не должна смешивать Markdown, backticks и shell-логику.\nCHECK-4: команда не должна иметь вложенный ssh с несколькими уровнями кавычек.\nCHECK-5: команда не должна печатать секреты.\nCHECK-6: если создаётся файл, сначала маленький безопасный шаг, потом проверка.\nCHECK-7: если команда длиннее 8 строк, её нельзя давать в интерактивный shell.\nCHECK-8: для edge-vm использовать debian@[PRIVATE_IP] и sudo.\nCHECK-9: для forum-prod использовать pve02 и ключ [SENSITIVE_PATH]\nCHECK-10: Corosync не трогать без отдельного плана и rollback.\n\n## Текущие важные факты\nInternal network: [PRIVATE_IP]/24.\nMigration config: migration: secure,network=[PRIVATE_IP]/24.\nCorosync remains on [PRIVATE_IP]/12/13.\nVM160 forum-prod is not in Proxmox nightly backup.\nVM130 edge-vm has secondary disk backup=0 risk.\n05_edge_compose_safe.tgz quarantined.\n\n## Правило для справочника\nНе генерировать большой справочник через интерактивную вставку.\nСледующий справочник делать только маленькими append-блоками или через файл, созданный вне интерактивного shell.\n\n11. Ошибка: считать offhost OK после failed rsync.\nЕсли rsync/scp упал, нельзя проверять latest offhost-файл без сверки имени.\nПроверка должна подтвердить именно новый архив, например дату 2026_06_30-00_20_59.\nСтарый OFFHOST_ZSTD_OK не закрывает новый backup.\n\n12. Ошибка: широкий secret-поиск по /opt/stacks дал шум.\nНе искать Cloudflare/SMTP сразу по /opt/stacks, MkDocs site, backups и HTML-отчётам.\nДля ротации сначала смотреть live-конфиги: /opt/npmplus/tls/certbot/renewal/*.conf и реальные credentials-файлы из них.\nЗначения секретов не печатать; выводить только пути, ключи и redacted-поля.\n\n13. Ошибка: SQL с одинарными кавычками внутри одинарной SSH-команды ломается.\nНельзя писать sqlite SQL вида j.type in ('object','array') внутри ssh '...'.\nДля sqlite через ssh использовать SQL без одинарных кавычек: char(36), length(j.atom), двойные внешние кавычки, либо отдельный файл.\nПеред запуском проверять, что команда не содержит конфликтующих уровней кавычек.\n\n14. Ошибка: путать контекст входа и узел выполнения.\nСтартовая точка оператора: root@pve01 / [PRIVATE_IP].\nedge-vm: ssh debian@[PRIVATE_IP], внутри использовать sudo.\npve02/pve03: ssh root@pve02 или ssh root@pve03.\nforum-prod: заходить через pve02, ключ [SENSITIVE_PATH]\nПеред каждой командой явно понимать: где выполняется локальная часть, где remote-часть, каким пользователем.\n\n15. Ошибка: повторно нарушено правило №13 после его добавления.\nСнова был использован SQL JSON-path в одинарных кавычках внутри ssh '...'.\nЗапрещено использовать в remote sqlite SQL фрагменты вида '$.key' или '.key'.\nДля JSON-path использовать только char(...), без одинарных кавычек внутри SQL.\nКоманду с ошибкой char(36)||.dns_provider считать битой и не использовать.\n\n16. Основополагающее правило: перед каждой командой ассистент обязан явно написать: Сверка с файлом ошибок: CHECK пройден.\nПосле этой строки можно давать только одну короткую команду или один логический блок команд.\nНельзя выдавать команды без предварительной сверки с этим файлом ошибок.\nНельзя продолжать после собственной ошибки без записи ошибки в этот файл.\n\n19. Уточнение формата: не писать отдельную строку сверки перед каждой командой в пакете.\nСверка должна быть внутри самой команды первым логическим блоком: grep правил из файла ошибок и RULE_CHECK_OK.\nРазрешено давать до трёх логически связанных команд сразу, по возможности в одну строку каждая.\nНе писать фразу \"ждём\" после команд; указывать только контрольные строки результата.\n\n20. Ошибка: широкий XenForo SMTP-поиск по src дал шум исходников.\nДля XenForo SMTP не grep-ать весь /src; сначала смотреть live src/config.php без значений, затем БД XenForo и xf_option/emailTransport.\nЗначения DB-паролей и SMTP-паролей не печатать.\n\n21. Ошибка: nested PHP php -r дал Parse error на forum-prod.\nКоманды 102-104 считать битыми: они не подтвердили и не изменили smtpSsl.\nНе продолжать длинные вложенные ssh+php -r с кавычками; для forum-prod лучше войти в VM или использовать короткие mysql/php команды без вложенного кода.\nРабочими считать: DB backup перед SMTP, SMTP password update, JSON postcheck; SMTP auth ещё не закрыт из-за CODE=530 на AUTH LOGIN.\n\n22. Ошибка: самодельный base64 PHP для SMTP auth сломан.\nКоманда 108 дала PHP Parse error на smtpHost и пустой proof-файл.\nНе использовать больше сгенерированные большие base64 PHP для SMTP/XenForo.\nДля XenForo SMTP использовать короткие mysql-запросы к xf_option и готовые штатные механизмы XenForo.\n\n23. Ошибка: MariaDB не поддержала cast('true' as json) в JSON_SET.\nКоманда 111 не изменила smtpSsl: OLD_SMTPSSL=false, SQL ERROR 1064, NEW_SMTPSSL=false.\nПеред боевым UPDATE сначала проверять JSON_SET синтаксис на тестовом JSON через SELECT.\nНе считать PHP_FPM_RELOAD_OK подтверждением изменения БД.\n\n24. Ошибка: exit 1 в interactive-check закрыл SSH-сессию.\nНельзя использовать { echo RULE_CHECK_FAIL; exit 1; } в командах, выполняемых прямо в интерактивном root@pve01 shell.\nПри failed-check использовать безопасный шаблон: if grep ...; then echo RULE_CHECK_OK; ...; else echo RULE_CHECK_FAIL; fi.\nНе делать fragile grep по точной строке с кавычками, если в файле ошибок строка содержит shell/SQL quoting.\n\n25. Ошибка анализа: не надо слепо включать smtpSsl=true при smtpPort=587.\nsmtpPort=587 обычно означает STARTTLS, а не implicit SMTPS.\nCODE=530 на AUTH LOGIN означает, что ручной checker не сделал STARTTLS; это не доказывает неверный SMTP-пароль.\nСначала проверять AUTH через openssl s_client -starttls smtp, без вывода секрета.\n\n26. Ошибка: openssl -crlf вместе с ручным CRLF ломает SMTP AUTH checker.\nКоманда 116 получила 535 Invalid base64 data in continued response после 334 Username.\nЭто указывает на битую base64-строку checker-а, а не на неверный SMTP-пароль.\nНе использовать -crlf, если команды уже отправляются с явным \\r\\n.\nДля 587 оставлять smtpSsl=false и проверять STARTTLS корректным checker-ом.\n\n27. Ошибка: Python heredoc внутри ssh сломал not_ok диагностику.\nКоманда 138 пересобрала dashboard, но диагностический Python-блок упал с NameError из-за кавычек.\nНе использовать heredoc Python внутри вложенного ssh; для dashboard verification использовать cat/grep JSON или простые команды.\n\n28. Ошибка: docker compose ls --format json | wc -l дал ложный COMPOSE_PROJECT_COUNT=1.\nПричина: JSON-вывод может быть одной строкой/структурой, wc -l не является счётчиком проектов.\nДля Docker Compose count использовать обычный docker compose ls --all + awk по строкам таблицы или JSON parser, не wc -l.\n\n29. Ошибка: monitoring compact status искал неверные имена health-файлов.\nФакт: npmplus-cert-expiry.txt не является подробным файлом; подробный файл называется npmplus-certificate-expiry.txt, а статусный alias npmplus-cert-expiry-health.txt.\nФакт: health-metrics.txt не обязан существовать как health-файл; health metrics могут публиковаться через node-exporter textfile collector.\nДля Prometheus targets проверять HTTP_CODE/RC явно, не оставлять пустой блок PROMETHEUS_TARGETS_LOCAL.\n\n30. Ошибка/инцидент: Prometheus API HTTP_CODE=000 после monitoring reference.\nФакт: в monitoring compact status curl к 127.0.0.1:9090 вернул HTTP_CODE=000, а raw inventory не показал контейнер prometheus среди monitoring containers.\nНельзя считать monitoring слой runtime-closed только по secret scan/reference proof; нужно отдельно проверить Prometheus container/API и обновить справочник по факту.\n\n31. Строгое правило: перед каждой командой сверяться с файлом ошибок и справочником.\nПеред любым действием команда обязана проверять /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md и текущий 31_HOMELAB_REFERENCE.md.\nКоманда обязана печатать ERROR_REGISTER_CHECK=OK и REFERENCE_CHECK=OK до основного действия.\nЕсли сверка не пройдена, команда не должна выполнять основное действие. Не использовать exit 1, чтобы не закрывать SSH-сессию.\n\n32. Ошибка: 153_PROMETHEUS_TARGETS_SETTLED_PROOF содержит SyntaxError из-за вложенного Python внутри SSH.\nФакт: Python -c внутри ssh потерял кавычки вокруг /tmp/prom-targets-settled.json, data, activeTargets, labels, job, health.\nФайл 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt нельзя считать валидным settled proof.\nДля таких проверок не использовать вложенный Python; использовать curl + grep/sed/awk либо отдельный локальный файл-скрипт.\n\n34. Ошибка операционного ввода: полный терминальный лог MobaXterm попал в shell.\nФакт: bash пытался выполнить строки Authenticating, MobaXterm banner, RULE_CHECK_OK, PROOF_OK и таблицы вывода как команды.\nТакие выводы не являются инфраструктурным состоянием; считать невалидными только соответствующие failed command attempts.\nДальше давать короткие команды и не вставлять обратно полный transcript в shell.\n\n33. Security finding: root authorized_keys на PVE-нодах имел права 777.\nФакт: inventory 158 показал [SENSITIVE_PATH] и backup-файлы с режимом 777 на pve01/pve02/pve03.\nНужно исправлять на [SENSITIVE_PATH]=700 и authorized_keys/authorized_keys.bak*=600, затем фиксировать before/after proof.\n\n35. Ошибка проверки: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED не доказал исправление прав.\nФакт: AFTER всё ещё показал 777 для [SENSITIVE_PATH] на pve01/pve02/pve03.\nВозможная причина: authorized_keys является symlink, а stat без -L показывает режим ссылки 777.\nНужно проверять stat -L целевого файла и считать 160 недостаточным proof.\n\n36. Quality check: Storage block needs integrity and pve03 capacity coverage review.\nФакт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.\nФакт: pve03 /mnt/staging observed at 77%, close to WARN threshold 80%, while disk-space.txt did not visibly list pve03_staging.\nBefore closing storage layer, verify block integrity and pve03 capacity coverage.\n\n37. Coverage gap: pve03_staging missing from disk-space health coverage.\nФакт: 169 showed pve03 /mnt/staging at 77%, close to WARN 80%, while disk-space.txt listed pve01_staging, pve02_staging and edge_immich_media only.\nBefore closing storage fully, add or document pve03_staging capacity monitoring.\n\n38. Quality check: Service Dependency Map block needs integrity review.\nФакт: terminal output around command 222 showed paste artifact near \"FAIL; fi\".\nBefore closing Service Dependency Map, verify 178_REFERENCE_SERVICE_DEPENDENCY_MAP_BLOCK.txt and the reference section for shell/log garbage or truncation.\n\n39. Ошибка проверки: post deep audit BAD_MARKER scan caught prose false positive.\nФакт: строка справочника с фразой No unresolved final REVIEW_REQUIRED... SyntaxError... была принята за BAD_MARKER.\nЭто не runtime/log мусор, а документированная фраза deep audit closure. В scan нужно исключать эту prose-строку.\n\n40. Ошибка/вывод: VM150 Mail cloud single-file upload failed with 413.\nФакт: Mail/WebDAV rejected one 6.6G vzdump file with 413 Request Entity Too Large.\nРешение: use VM150 recurring chunked script with 512M parts and download verification.\n\n41. Ошибка: post-backup-pass-slo-reconciliation checked pve01 router health on edge-vm.\nФакт: first reconciliation wrote STATUS=ERROR because router-running-config-mail-cloud.txt is on pve01, not edge-vm.\nFix: build reconciliation from pve01 and edge-vm health files by correct owner.\n\n## ASSISTANT_COMMAND_BATCHING_RULE_20260630\n- Rule: combine related infrastructure checks, proof generation, reference updates and validation scans into one safe batch command when practical.\n- Do not split work into many small commands when the operations are low-risk, read-only, or naturally part of one evidence block.\n- Still keep mandatory preflight checks, short bounded commands, no large paste, no here-docs, no interactive shell traps, and no sensitive values in output.\n- Split into smaller commands only when safety, rollback, quoting risk, or troubleshooting clarity requires it.\n\n## HOME_PORTAL_BASE64_APPLY_FAILURE_20260630\n- Prior home-portal apply attempt 312 failed because a large base64 Python payload decoded as invalid UTF-8.\n- Rule: avoid large opaque base64 script payloads for homelab changes; use transparent bounded shell/perl/python commands instead.\n\n## HOMELAB_COMMAND_SAFETY_HARDENING_20260630\n- Rule: do not use large opaque payloads, generated base64 scripts, or hidden multi-line script blobs for homelab changes.\n- Use transparent bounded shell/perl/python commands, or create a temporary readable script and validate it before execution.\n- Any script used for changes must pass a syntax/parse check before it is allowed to modify files or restart services.\n- Any Traceback, UnicodeDecodeError, SyntaxError, Permission denied, command-not-found, nonzero critical RC, or failed write marker must force REVIEW or FAILED, never OK.\n- Success requires both runtime health and content-specific post-checks: expected files changed, expected cards exist, duplicates absent, service/container alive, and no new critical log pattern.\n- HTTP 200 alone is not a success condition for configuration changes.\n- If an apply step fails, the next command must explicitly record the failed attempt and then run a corrected transparent apply command.\n\n## HOME_PORTAL_API_ERROR_CLOSURE_RULE_20260630\n- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200.\n- If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed.\n- External informational widgets such as weather/Open-Meteo must not block the service launcher portal.\n\n## HOME_PORTAL_COMPLEX_QUOTING_FAILURE_20260630\n- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('.\n- Cause: command was too complex and fragile due to nested shell/perl/python quoting.\n- Rule: do not use deeply nested one-liners for config edits; first run short state checks, then apply small transparent edits.\n\n## HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RULE_20260630\n- Rule: Homepage cleanup validation must check active config files only, not logs or backup files.\n- Active files are services.yaml, bookmarks.yaml, widgets.yaml, settings.yaml and docker.yaml under /opt/stacks/homepage/config.\n- Matches in /logs/ or *.bak* are historical evidence only and must not make cleanup REVIEW unless the active config still contains the target.\n\n## HOME_PORTAL_LINK_VALIDATION_NPMPLUS_DEFAULT_RULE_20260630\n- Rule: Homepage link validation must not treat HTTP 200 as success if the body is the NPMPlus default “Congratulations” / host-not-set-up page.\n- Green siteMonitor dots can be false positives when NPMPlus returns its default vhost page.\n- A portal card is openable only when DNS, reverse-proxy host mapping, and target content are all valid.\n\n## ROUTER_CLI_STDIN_APPLY_FAILURE_20260630\n- Context: attempted Netcraze router ACL apply through SSH stdin/multiline for Homepage Moscow Router monitor fix.\n- Evidence: 391_MOSCOW_ROUTER_EDGE_VM_WEBADMIN_ACL_APPLY_PROOF.txt.\n- Failure: STDIN_SHOW_OK=no, APPLY_SKIPPED=stdin_show_failed, CONFIG_HAS_EDGE_5080=no, CONFIG_HAS_EDGE_5083=no.\n- Rule: do not apply Netcraze config through stdin/multiline SSH; prove CLI input mode first.\n\n## ROUTER_CLI_PROBE_STDIN_CONSUMPTION_ERROR_20260630\n- Context: ACL syntax read-only probe loop executed only one command because ssh consumed the loop stdin.\n- Evidence: 393_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_PROOF.txt showed COMMAND_COUNT=1.\n- Fix pattern: use ssh -n or redirect SSH stdin away from the command-list loop for all future SSH-in-loop probes.\n\n## ROUTERBACKUP_READONLY_CLI_PERMISSION_LIMIT_20260630\n- Context: routerbackup user can run show running-config but cannot execute configuration commands such as access-list.\n- Evidence: 394_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_FIXED_PROOF.txt.\n- Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli].\n- Rule: do not attempt ACL/config changes as routerbackup; use admin/operator router access or router Web UI for write changes.\n\n## PROOF_SUMMARY_EXTRACTION_BLANK_20260630\n- Context: proof 395 tried to read COMMAND_COUNT/ZERO_RC_COUNT from proof files 393/394, but those summary fields were printed to console and were not persisted inside the proof files.\n- Evidence: 395_ROUTER_CLI_ERROR_REGISTER_UPDATE_PROOF.txt has blank P393_COMMAND_COUNT/P394_COMMAND_COUNT/P394_ZERO_RC_COUNT fields.\n- Rule: when creating follow-up proof summaries, derive counts from persisted proof body lines such as CMD_INDEX and CMD_RC, or write summary fields into the proof file itself.\n\n## ROUTER_ACL_UI_RULE_DELETE_AND_RESTORE_20260701\n- Context: while moving Moscow Router monitor rules to _WEBADMIN_Bridge1, two _WEBADMIN_Bridge0 Nextcloud Talk TURN rules were accidentally removed.\n- Deleted/restored rules: Home [PRIVATE_IP]/24 to Nextcloud [PRIVATE_IP] TCP/UDP 3478.\n- Verification proof: 399_ROUTER_ACL_RESTORE_AFTER_UI_PROOF.txt.\n- Rule: after manual router Web UI ACL edits, compare _WEBADMIN_Bridge0, _WEBADMIN_Bridge1 and _WEBADMIN_GigabitEthernet1 before continuing.\n\n## ROUTER_MONITOR_NODE_LOOKUP_PROBE_BUG_20260701\n- Context: proof 400 tested Moscow router HTTPS monitor from Homepage container with custom Node lookup callback.\n- Failure: NODE_HTTPS_RESOLVE_ERROR=ERR_INVALID_IP_ADDRESS and NODE_HTTPS_RESOLVE_INSECURE_ERROR=ERR_INVALID_IP_ADDRESS.\n- Impact: Node HTTPS resolve subtest is invalid; curl --resolve results remain valid.\n- Rule: do not rely on custom Node lookup callback until separately tested; prefer curl --resolve or simple direct Node HTTP checks.\n\n## ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701\n- Context: Moscow Router Homepage monitor after ACL fix.\n- Evidence: proofs 399, 400, 401.\n- Observed: edge-vm TCP to [PRIVATE_IP]:5080/5083 is OK, but router HTTP/HTTPS returns 403 for all tested paths.\n- Impact: Homepage siteMonitor cannot be made green via direct router HTTP URL until router web service allows the Bridge1/Proxmox/protected segment or an explicit safe monitor endpoint is used.\n- Rule: do not keep changing ACL for this symptom; ACL is already correct. Fix service access scope or use a dedicated monitor endpoint.\n\n## PY_COMPILE_PYC_PERMISSION_ERROR_20260701\n- Context: installing edge-vm Moscow router health endpoint.\n- Mistake: assistant used `python3 -m py_compile` on a script installed under /usr/local/sbin as root.\n- Actual impact: non-root syntax check attempted to create /usr/local/sbin/__pycache__ and failed with Permission denied.\n- Recovery: validate syntax with `ast.parse` instead of py_compile, because it does not write .pyc files.\n- Rule: do not use py_compile against root-owned system paths from an unprivileged user.\n\n## ROUTER_HOME_IP_DOCKER_ROUTE_CONFLICT_20260701\n- Context: Moscow Router Homepage siteMonitor attempt using http://[PRIVATE_IP]:5080.\n- Evidence: proof 405 showed route to [PRIVATE_IP] via Docker bridge and Homepage container ECONNRESET.\n- Impact: do not use [PRIVATE_IP] directly as Homepage siteMonitor from edge-vm.\n- Mitigation: use dedicated edge-vm health endpoint that checks router TCP [PRIVATE_IP]:5080 and returns HTTP 200/503 for Homepage.\n\n## HOMEPAGE_ROUTER_MOSCOW_YAML_TITLE_SHAPE_MISMATCH_20260701\n- Context: applying Moscow Router Homepage siteMonitor health endpoint.\n- Mistake: assistant apply script matched only property-style cards with `title`, but active Homepage YAML can use service-name-key style like `- Service Name:`.\n- Evidence: proof 407 returned YAML_CHANGE_COUNT=0 and APPLY_SKIPPED=unexpected_match_count while health endpoint proof 406 was OK.\n- Recovery: use YAML-aware updater that handles both property-style and service-name-key style cards.\n\n## HOMEPAGE_ROUTER_MOSCOW_APPLY_SCRIPT_SYNTAX_ERROR_20260701\n- Context: applying Moscow Router Homepage health endpoint.\n- Mistake: assistant generated Python script with invalid f-string escaping.\n- Evidence: proof 408 showed SyntaxError in /tmp/homepage-router-moscow-apply-fixed.py.\n- Actual impact: YAML was not changed, so Homepage green dot could not appear.\n- Recovery: stop applying blind updater; first discover exact active YAML shape, then apply minimal YAML-aware edit.\n- Rule: validate generated Python syntax locally before scp/remote execution; avoid f-strings in generated maintenance scripts.\n\n## FORUM_PROD_VM160_FIRST_BOOT_SSH255_20260701\n- Context: clean rebuilt VM160 first boot.\n- Symptom: PING_OK=yes and TCP22_OK=yes, but SSH_RC=255.\n- Rule: do not proceed with OS baseline until SSH failure is diagnosed; likely old known_hosts key or cloud-init/root-key issue.\n\n## FORUM_PROD_VM160_NESTED_HOSTNAME_PROOF_QUOTING_20260701\n- Context: VM160 first SSH proof after rebuild.\n- Issue: command substitution $(hostname) inside nested ssh was expanded on pve02 before entering VM160.\n- Impact: VM_HOSTNAME=pve02 in proof 426 is not valid VM identity evidence.\n- Rule: for nested SSH identity checks, run literal hostname commands without local command substitution.\n\n## FORUM_PROD_VM160_SWAP_PROOF_QUOTING_20260701\n- Context: VM160 swapfile proof 429.\n- Issue: nested command substitution expanded on pve02, producing invalid SWAPFILE/FSTAB proof lines.\n- Impact: proof 429 is not valid closure evidence even though swap was active.\n- Rule: avoid nested $(...) in VM SSH proofs; use literal remote commands and clean proof.\n\n## FORUM_UPLOAD_ARCHIVES_VM_VERIFY_QUOTING_20260701\n- Context: proof 446 copy/check archives inside VM160.\n- Issue: nested SSH quoting expanded shell variables incorrectly, producing gzip checks against empty .gz and blank TAR_TOP lines.\n- Impact: SCP_TO_VM_RC=0 and file size list are valid, but GZIP/TAR proof lines in 446 are invalid.\n- Rule: verify VM archive integrity locally inside forum-prod, not through nested quoted loops.\n\n## FORUM_PROD_BULK_IMPORT_PHP85_EMPTY_FRONTEND_20260701\n- Context: rebuilt VM160, five XenForo forums imported in bulk on PHP 8.5.7.\n- Symptom: all five forums returned HTTP 200 with empty body; XenForo logged Template public:PAGE_CONTAINER/forum_list is unknown.\n- Additional failed approach: proof 480 ran xf-dev:rebuild-caches on CodeVipe and returned RC=1, not a valid fix.\n- Decision: stop bulk repair, treat current VM160 as lab, find a working CodeVipe recipe first, then destroy/rebuild cleanly for all five forums.\n- Rule: no more mass rebuild/repair commands across all forums until single-forum CodeVipe recipe is proven.\n\n## XENFORO_REQUIREMENTS_DIRECT_DOWNLOAD_403_20260701\n- Context: Tried to download XenForo requirements ZIP directly from xenforo.com inside forum-prod.\n- Issue: curl returned HTTP 403; unzip then failed because the ZIP was not downloaded.\n- Evidence: proof 491 showed DOWNLOAD_RC=22, ZIP_LIST_RC=9, UNZIP_RC=9.\n- Impact: proof 491 is not a valid server compatibility test.\n- Rule: use operator-uploaded xenforo23-requirements-test.zip from /root/forum-upload instead of relying on direct curl download.\n\n## PVE01_UNZIP_MISSING_FOR_XF_REQUIREMENTS_VERIFY_20260701\n- Context: proof 492 tried to inspect uploaded xenforo23-requirements-test.zip on pve01.\n- Issue: pve01 does not have unzip installed, so ZIP_TEST_RC=127 was not a ZIP integrity result.\n- Impact: proof 492 confirmed file presence only, not archive validity.\n- Rule: verify ZIP with Python zipfile or inside forum-prod rather than installing unzip on Proxmox just for inspection.\n\n## FRESH5_DEPLOY_SUCCESS_20260701\n- Context: VM160 was destroyed/recreated, then five XenForo forums were deployed from fresh ZIP backups.\n- Result: proof 513 confirms all five forums locally healthy.\n- Note: proof 511 initially failed dsmods path detection; proof 512 fixed dsmods by locating public_html via src/XF.php marker.\n- Rule: future restores should preserve internal_data/code_cache or rebuild compiled template cache before smoke testing.\n\n## NPMPLUS_SQLITE_PASTE_FAILURE_20260701\n- Context: NPMplus SQLite inspection command was too complex and a Python fragment was pasted into bash.\n- Issue: shell entered multiline prompt and produced syntax errors.\n- Impact: do not trust that SQLite inspection attempt.\n- Rule: avoid nested SSH heredoc/Python for this task; use simpler commands, generated nginx configs, or uploaded/local script files.\n\n## NPMPLUS_PUBLISH_SCRIPT_WRONG_ENV_SOURCE_525_20260701\n- Context: proof 525 tried to create forum proxy hosts in NPMplus.\n- Issue: script sourced /opt/npmplus/.env, but INITIAL_ADMIN_EMAIL/PASSWORD were not present there; they are in the npmplus container env.\n- Impact: no forum proxy hosts were created by proof 525.\n- Rule: read NPMplus API login values from docker inspect env internally, never print them.\n\n## NPMPLUS_API_LOGIN_MISSED_HTTPS81_AND_JSON_ESCAPE_20260701\n- Context: NPMplus API login attempts in proofs 526/527 failed.\n- Issue: commands missed the likely HTTPS admin endpoint on port 81 and built JSON by string interpolation instead of jq escaping.\n- Impact: no proxy hosts were created by 526/527.\n- Rule: for NPMplus API use jq-generated JSON and test https://127.0.0.1:81/api/tokens with -k before publish.\n\n## NPMPLUS_API_AUTH_UNAVAILABLE_MANUAL_ROUTES_20260701\n- Context: NPMplus API login attempts failed even though the forum VM and edge-to-forum connectivity are healthy.\n- Issue: API credentials from container initial env are not accepted by current NPMplus API.\n- Impact: do not use NPMplus API for this publish path.\n- Rule: publish forum routes through backed-up manual nginx proxy_host configs, then issue proper certificates after DNS points to edge.\n\n## EDGE_CERTBOT_TOKEN_FILE_PERMISSION_535_20260701\n- Context: proof 535 attempted DNS-01 certificate issue for five forum domains.\n- Issue: /tmp/forum_cf_token.env was root-owned mode 600, so debian user could not source it directly.\n- Impact: certbot ran without token and all certificate attempts failed; DNS cutover must not proceed from proof 535.\n- Rule: read Cloudflare token via sudo without printing it, then create temporary certbot credentials file and remove it after use.\n\n## EDGE_CERTBOT_FAILED_DNS_CUTOVER_ABORTED_20260701\n- Context: edge manual routes are working locally, but certbot DNS-01 certificate issue failed in proof 538.\n- Issue: all five certbot attempts returned RC=1; proof 539 correctly aborted DNS cutover because edge cert precheck was not ready.\n- Impact: Cloudflare DNS records were not changed by proof 539; public DNS still points to old IP 87.236.18.45.\n- Rule: inspect certbot logs and Cloudflare token zone permissions before retrying certificate issuance or DNS cutover.\n\n## CLOUDFLARE_TOKEN_NO_ZONE_ACCESS_20260701\n- Context: Cloudflare token discovered on edge verified as a token but could not access forum zones.\n- Evidence: proof 542 showed all five zones missing and DNS record create probes failed.\n- Impact: certbot DNS-01 failed and DNS cutover was correctly aborted; public DNS still points to old hosting.\n- Rule: install a new Cloudflare token with Zone Read and DNS Edit for the five forum zones before retrying certs/DNS.\n\n## FORUM_PUBLICATION_FINAL_SUCCESS_20260701\n- Context: five XenForo forums were rebuilt on VM160, routed through edge NPMplus, issued Let’s Encrypt certs using a corrected Cloudflare token, and cut over in DNS.\n- Result: final public proof 546 passed.\n- Important corrections: initial NPMplus API route failed; manual Nginx routes were used. Initial Cloudflare token had no zone access; corrected token passed zone/DNS probe.\n\n## FINAL_SNAPSHOT_NAME_TOO_LONG_556_20260701\n- Context: proof 556 final health gate passed for all five public forums.\n- Issue: Proxmox snapshot failed because generated snapshot name exceeded 40 characters.\n- Evidence: qm snapshot returned snapname value may only be 40 characters long.\n- Impact: forum health was OK, but proof 556 snapshot step was not completed.\n- Fix: rerun snapshot with short name.\n\n## CLOUDFLARE_DNS_AUDIT_TXT_SUBSTRING_GAP_20260701\n- Context: prior Cloudflare DNS audit checked web A/AAAA/CNAME state for root/www records.\n- Issue: it did not deeply inspect TXT/SPF contents for old provider IP substrings.\n- Evidence: user screenshot showed SPF TXT containing old 87.236.18.* value.\n- Impact: web routing is OK, but mail-related DNS may still contain stale provider data.\n- Rule: perform full DNS record audit including TXT/SPF/DKIM/DMARC/MX/CDN and substring search for old IPs before DNS cleanup.\n\n## CLOUDFLARE_DNS_CLEANUP_DRYRUN_QUOTE_563_20260701\n- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt.\n- Impact: proof 563 is invalid and no DNS cleanup was performed by it.\n- Rule: use copied scripts for Cloudflare DNS mutation instead of fragile inline nested quotes.\n\n## SPF_DUPLICATE_AFTER_565_20260701\n- Context: SPF cleanup command 565 attempted to replace stale SPF records.\n- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained.\n- Impact: domains temporarily had duplicate SPF records, which is invalid for mail validation.\n- Fix: delete all SPF TXT records for the five forum zones, then create exactly one v=spf1 -all TXT per zone.\n\n## MSMTP_SECRET_SOURCE_PARSE_ERROR_576_20260701\n- Context: proof 576 installed msmtp but sendmail auth test failed.\n- Issue: [SENSITIVE_PATH] was sourced as shell, but SMTP password contained shell-special characters; source failed and auth used an invalid/truncated secret path.\n- Impact: msmtp package installed, but mail sending was not proven working.\n- Rule: never source SMTP secret files containing arbitrary passwords; store password base64 and use msmtp passwordeval helper accessible to www-data.\n\n## FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701\n- Context: attempted to fix msmtp config with passwordeval helper.\n- Issue: one or more sendmail/PHP mail tests still failed.\n- Impact: XenForo mail sending is not yet proven.\n- Next step: verify SMTP credential/password and provider policy for noreply@pvepro.ru.\n\n## SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701\n- Context: SMTP password was exposed in terminal output during failed msmtp setup.\n- Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line.\n- Impact: treat that SMTP password as compromised.\n- Required action: rotate the mailbox/app password in the mail provider panel before any further SMTP testing.\n- Rule: never store arbitrary passwords as shell-sourceable KEY=value files; use non-printing secret storage only.\n\n## NETBIRD_MAILCOW_REPORTED_DOWN_AFTER_SMTP_INCIDENT_20260701\n- Context: user reported NetBird and Mailcow became unavailable after forum mail-transport work.\n- Impact: treat as active incident until service reachability and container/VM state are proven.\n- Rule: do not change DNS, firewall, proxy, mail, NetBird, or Mailcow configs before read-only triage proof.\n\n## FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701\n- Context: forum-prod SMTP/msmtp test used invalid or compromised credentials and triggered Mailcow netfilter warnings/ban for 95.84.154.183.\n- Impact: forum mail transport is not proven and must remain disabled until SMTP password is rotated and tested once carefully.\n- Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth.\n- Rule: do not test SMTP auth again until a fresh rotated mailbox/app password is available.\n\n## FORUM_OLD_CODEVIPE_CLOUD_BACKUP_STALE_AFTER_FRESH5_20260701\n- Context: pve02 codevipe-cloud-backup.timer existed before fresh five-forum rebuild.\n- Issue: old script points to /root/backups/codevipe and old remote path app/codevipe/forum-prod/local.\n- Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory contains old codevipe-public/codevipe.sql files, not current five-forum archive.\n- Impact: old timer must not be treated as valid current backup for all five forums.\n- Fix: create a new forum-prod fresh5 cloud backup using existing pve02 Mail-cloud rclone crypt remotes.\n\n## FORUM_RESTORE_DRILL_610_STATUS_FLAG_BUG_20260701\n- Context: restore drill proof 610 downloaded Mail.ru Cloud backup, verified split part SHA256, reconstructed archive, matched full archive SHA256, and confirmed all five forum DB/file archives in tar.\n- Issue: final RESTORE_DRILL_OK was set to no because the script tested an unassigned shell variable RCLONE_REMOTE_PVE02_MAIL_01_RC.\n- Impact: proof 610 final status flag is invalid, but its detailed integrity checks are valid.\n- Fix: rerun corrected restore validation against the downloaded cloud backup artifacts and produce proof 612.\n\n## FORUM_XENFORO_MAIL_SMOKE_TEST_FAILED_20260701\n- XenForo-level mail smoke test did not return success for all five forums.\n- Check proof 623 and msmtp log before retrying.\n\n## XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701\n- Context: custom XenForo mail smoke test 623 failed with Could not open input file /root/scripts/623_xenforo_mail_smoke.php.\n- Cause: the script was run as www-data from /root/scripts; /root is not traversable by www-data.\n- Impact: proof 623 is invalid and should not be used to judge mail delivery.\n- Superseding evidence: user ran XenForo built-in outgoing email test; message was delivered to Yandex but placed in Spam.\n- Next step: investigate deliverability/authentication headers and pvepro.ru SPF/DKIM/DMARC/PTR.\n\n## FORUM_NEXT_CHAT_KNOWN_ERRORS_AND_CLOSED_INCIDENTS_20260701\n\n### Closed / classified incidents\n- SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701:\n - A previous bad command sourced a raw SMTP secret and printed it.\n - Treat old password as compromised.\n - Later persistent SMTP was rebuilt using safe files and verified.\n - Never print or package secrets.\n\n- XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701:\n - Custom mail proof 623 failed with \"Could not open input file\".\n - Cause: PHP was run as www-data from /root/scripts, but /root is not traversable by www-data.\n - Impact: proof 623 is invalid and must not be used to judge mail delivery.\n - Superseded by user-observed built-in XenForo test and Yandex header proof.\n\n- FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701 / FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701:\n - Earlier SMTP attempts failed and triggered mailcow netfilter ban for forum public IP.\n - Ban was removed.\n - Persistent msmtp transport was later enabled and verified.\n - Mailcow and NetBird remained reachable after final tests.\n\n- SPF_DUPLICATE_AFTER_565_20260701:\n - Earlier SPF cleanup created duplicate SPF records.\n - Fixed by deleting duplicates and recreating exactly one SPF per forum domain.\n - Final forum domains use v=spf1 -all because sending uses noreply@pvepro.ru.\n\n- FORUM_RESTORE_DRILL_610_STATUS_FLAG_BUG_20260701:\n - Restore drill 610 had a status flag bug despite successful detailed checks.\n - Corrected restore drill 612 passed.\n\n- FORUM_DBTECH_TOR_TIMEOUT_CLASSIFIED_20260701:\n - One historical timeout row per forum to check.torproject.org.\n - DBTech/Security active, but dbtech_security_tornodes=0.\n - Classified as external timeout noise, not runtime failure.\n\n### Current non-blocking items\n- Yandex placed built-in XenForo test mail in Spam despite SPF pass and DKIM pass.\n- Classification: deliverability/reputation/content filtering, not server failure.\n- Recommended manual action: click “Не спам!” and add/check pvepro.ru in Yandex Postmaster.\n\n### Safety rules for next chat\n- Do not print secrets.\n- Do not download or upload:\n - [SENSITIVE_PATH]\n - /etc/msmtprc\n - /etc/msmtp/*\n - rclone configs\n - Cloudflare tokens\n - DB dumps\n - VM disks\n - backup archives\n- For handoff, only share the two truth files and selected non-secret proof files.\n\n## PARKED_DOMAINS_STAGE4_DNS01_PREFLIGHT_FAILED_20260701\n- Context: parked-domain public apply proof 634.\n- Issue: DNS-01 based apply was attempted before a clear CAN_APPLY=yes preflight result was visible in terminal output.\n- Observed: stage4 stopped safely with preflight_failed; CERTBOT_PRESENT=no, CERTBOT_DNS_CLOUDFLARE_PLUGIN=no, and CF_ZONE_ACCESS=no for newfi.ru, hapusya.ru and kingofwolk.ru.\n- Impact: no parked-domain NPMplus route/certificate changes were applied by stage4; public HTTPS validation still failed certificate verification.\n- Rule: do not use Cloudflare DNS-01 for these parked domains unless a token with explicit zone access is installed; because DNS already points to edge, prefer a copied-script HTTP-01 flow with temporary ACME challenge routes and content-specific validation.\n\n## PARKED_DOMAINS_STAGE5_BASH_LOCAL_SETU_BUG_20260701\n- Context: parked-domain HTTP-01 apply proof 635.\n- Issue: edge script used Bash `local id=\"$1\" ... conf=\"$WORK/.../$id.conf\"` and `local host=\"$1\" ... tmp=\"$WORK/.../$host.html\"` under `set -u`; dependent variables are not safe inside the same local assignment command.\n- Observed: `id: unbound variable`, `source can not be empty`, then `host: unbound variable`; certbot did not run, final HTTPS routes were not installed.\n- Impact: local parked page remained OK; public HTTPS remained not closed.\n- Rule: split dependent Bash local assignments into separate statements, syntax-check copied scripts before execution, and keep content-specific validation for parked routes.\n\n## PARKED_DOMAINS_STAGE6_TEMP_HTTP_ROOT_VALIDATION_BUG_20260701\n- Context: parked-domain HTTP-01 fixed apply proof 636.\n- Issue: temporary HTTP-01 route validation checked HTTP `/` for parked marker, but expected behavior before certificates is root HTTP 301 to HTTPS.\n- Observed: LOCAL_HTTP_VALIDATE returned HTTP_CODE=301 MARKER_OK=no, script rolled back its temporary route configs and did not run certbot.\n- Impact: local parked page remained OK; public HTTPS remained not closed.\n- Rule: validate `/.well-known/acme-challenge/` for HTTP-01 readiness, not HTTP `/`; also scan all active NPMplus nginx config dirs for unmanaged domain conflicts before writing routes.\n\n## PARKED_DOMAINS_STAGE7_NEWFI_ROOT_ACME_REDIRECT_20260701\n- Context: parked domains HTTP-01 Stage7 stopped before certbot because ACME validation path returned 301 for root newfi.ru while www.newfi.ru, hapusya.ru, www.hapusya.ru, kingofwolk.ru and www.kingofwolk.ru returned 200.\n- Impact: no final parked-domain HTTPS routes/certificates were applied by Stage7; temporary route configs were rolled back.\n- Rule: do not run another apply stage until active NPMplus nginx route selection for newfi.ru is diagnosed with nginx -T and Host-header probes.\n\n## PARKED_DOMAINS_STAGE8_DEFAULT_HTTP_TO_HTTPS_REDIRECT_20260701\n- Context: parked-domain route autopsy proof 638.\n- Finding: no active NPMplus config contains newfi.ru/hapusya.ru/kingofwolk.ru, but unknown HTTP hosts and ACME paths return 301 to HTTPS from NPMplus default routing.\n- Impact: HTTP-01 cannot work through the default server. A dedicated temporary ACME server block must be proven before certbot; do not run another certbot apply without a successful ACME-path probe for all six hostnames.\n- Rule: after every parked-domain temp route attempt, rollback the route and prove rollback before continuing.\n\n## PARKED_DOMAINS_STAGE9_NEWFI_EXACT_PROBE_NEEDED_20260701\n- Context: parked domains HTTP-01 probe stage9 installed one combined temporary server block for all six hostnames.\n- Issue: five hostnames returned ACME probe 200, but root newfi.ru still returned 301.\n- Impact: do not continue certbot/apply until exact newfi.ru route behavior is isolated.\n- Rule: run a temporary exact newfi.ru probe block and roll it back before any HTTP-01 apply.\n\n## PARKED_DOMAINS_STAGE10_EXACT_NEWFI_PROBE_OK_20260701\n- Context: parked-domain Stage10 proof 640.\n- Result: exact server block for newfi.ru on NPMplus intercepted both root and ACME paths with HTTP 200 and custom X-Parked-Probe markers.\n- Stage9 combined server_name block passed 5/6 but failed bare newfi.ru, so final HTTP-01 cutover must use separate exact per-host server blocks.\n- Rule: do not use one combined parked-domain server_name block for these domains; use exact host blocks and content-specific validation.\n\n## PARKED_DOMAINS_STAGE11_LOW_ID_EXACT_ROUTE_FAILED_20260701\n- Context: parked-domain Stage11 proof 641.\n- Result: exact per-host route files 700-705 passed nginx -t but bare newfi.ru still hit HTTP->HTTPS redirect on ACME path.\n- Earlier Stage10 proof showed exact newfi.ru route works when placed at high id 998.\n- Hypothesis: file/include priority or NPMplus generated ordering makes low id 700 unsuitable for bare newfi.ru in this runtime.\n- Rule: do not retry parked-domain HTTP-01 with low id 700-705. First prove fixed high-id exact blocks, then apply with those exact ids.\n\n## PARKED_DOMAINS_SWITCH_TO_DNS01_AFTER_HTTP01_FAILURES_20260701\n- Context: parked domains newfi.ru, hapusya.ru and kingofwolk.ru.\n- HTTP-01 attempts hit NPMplus default HTTP-to-HTTPS redirect/include-order problems.\n- Stage12 cleanup confirmed temp files 980-985 absent and nginx reload OK.\n- Rule: do not continue HTTP-01 for these parked domains; use DNS-01 with a dedicated Cloudflare token that has Zone Read and DNS Edit for the three zones.\n\n## PARKED_DOMAINS_STAGE15_FALSE_LOCAL_VALIDATE_ROLLBACK_20260701\n- Context: Stage15 DNS-01 succeeded and public validation showed parked page for most hosts, but local edge curl --resolve validation returned CURL_RC=60 and triggered rollback.\n- Impact: do not rerun Stage15 as-is.\n- Rule: for parked domains finalization, use existing DNS-01 certificates and validate from pve01 public HTTPS with content marker; do not use the failed local --resolve TLS check as the closure gate.\n\n## DOMAIN_ACTIONS_STAGE18_PLACEHOLDER_AND_RENEWAL_PROOF_20260701\n- Context: after parked domains and portfolio audit, operator requested removing domain enumeration from placeholder and checking certificate renewal paths.\n- Rule: Stage18 must not change DNS records or NPMplus route files. It may update placeholder HTML, run existing forum renewal script once for runtime proof, and run certbot dry-run for parked certificates using existing DNS-01 hooks.\n\n## GRAM1_ROOT_WWW_PLACEHOLDER_CLOSE_STAGE19_20260701\n- Context: operator requested gram1.ru root/www to use the existing placeholder page.\n- Rule: only root gram1.ru and www.gram1.ru may be routed. Existing gram1.ru subdomain NPMplus routes must not be changed.\n- Rule: use DNS-01 with a dedicated gram1 Cloudflare token; do not use HTTP-01.\n- Stage18 parked dry-run note: kingofwolk.ru dry-run hit Let's Encrypt rateLimited/service-busy after active certificate issuance and public HTTPS were already OK. Treat as transient external dry-run failure, not route/cert closure blocker.\n\n## PVEPRO_EDGE_LANDING_STAGE21_20260701\n- Context: pvepro.ru root/www currently point to the Mailcow VPS and return Mailcow with TLS hostname mismatch.\n- Decision: because pve01 has no SSH key access to the external VPS, do not modify the VPS. Instead move only pvepro.ru and www.pvepro.ru A records to edge and serve an edge landing page.\n- Safety rule: do not change mail.pvepro.ru, nb.pvepro.ru, MX, SPF, DKIM, DMARC or forum SMTP configuration.\n- Use a dedicated Cloudflare token for pvepro.ru. Do not print token values.\n\n## PVEPRO_STAGE21_LANDING_SERVICE_FAILED_20260701\n- Context: Stage21 stopped safely before DNS/cert/NPMplus route because local pvepro landing service returned CURL_RC=7 on 127.0.0.1:18089.\n- Impact: DNS root/www pvepro.ru remained on 185.139.214.215 and no route/cert changes were applied.\n- Rule: Stage22 must fix and prove the local landing service first, then proceed to DNS-01 cert, NPMplus route and root/www DNS only after local marker PVEPRO_LANDING_OK is reachable.\n\n## PVEPRO_STAGE22_VALIDATION_AND_COPY_GUARD_20260701\n- Context: Stage22 succeeded on edge-side landing, cert, NPMplus route and Cloudflare A upserts, but wrapper had a non-critical scp wildcard failure due __pycache__ and public validation used cached DNS for root pvepro.ru.\n- Mistakes recorded: do not scp scripts/* after local py_compile because __pycache__ may be present; do not continue after critical scp failure in future apply stages; do not classify PVEPro landing as NetBird just because it contains a link text; use Cloudflare authoritative A records and --resolve validation when DNS propagation may lag.\n- Stage23 must be read-only for infrastructure and may close only if edge route, Cloudflare A records, HTTPS landing marker, and mail/nb unchanged checks pass.\n\n## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701\n- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router.\n- Current limitation: there is no safe private remote access path to the dacha router yet.\n- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet.\n- Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists.\n\n## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701\n- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router.\n- Current limitation: there is no safe private remote access path to the dacha router yet.\n- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet.\n- Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists.\n\n## TAFTAUTO_WG_STAGE24_SECRET_AND_IMPORT_RULE_20260701\n- Context: configuring private management path for taftauto.ru dacha router.\n- Rule: do not print WireGuard private keys, PSK, or client config body in proof/chat.\n- Rule: generated client config may be stored only as root/debian mode 600 file and must be imported into the Netcraze router UI manually.\n- Rule: do not open router public admin or certificate deployment until WireGuard handshake is proven.\n\n## TAFTAUTO_WG_PSK_ROTATION_SCRIPT_BROKE_DATAPLANE_20260702\n- Assistant gave unsafe PSK rotation flow; after profile reimport dacha interface changed from Wireguard0 to Wireguard1 and data-plane broke until security-level, ACL and routes were restored on Wireguard1.\n- Proof 666 is superseded because it showed PUBLIC_SSH_22_STILL_OPEN.\n- Final fixed proof: /root/evidence/667_TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702_PROOF.txt\n\n## TAFTAUTO_CERTBOT_CLOUDFLARE_PLUGIN_MISSING_20260702\n- Certbot on edge rejected --dns-cloudflare-* arguments: dns-cloudflare plugin missing/not loaded.\n- No taftauto certificate issued in failed run; no DNS A records or NPMplus routes changed.\n\n## HOMEPAGE_SERVICES_YAML_BAD_INDENT_20260702\n- Assistant inserted Homepage services.yaml entries with wrong indentation; YAMLException bad indentation at line 227.\n- Fix: rebuild SmartApe under External Homelabs with 2-space item indentation, remove Router Moscow, rebuild Sites category.\n\n## HOMEPAGE_SERVICES_YAML_BAD_INDENT_REPAIR_20260702\n- Previous apply broke services.yaml indentation and did not follow YAML-aware rule.\n- Repair uses latest valid services.yaml backup, PyYAML parse/dump, validates before restart.\n\n## TAFTAUTO_CERTBOT_DRYRUN_BAD_FLAG_20260702\n- Assistant used unsupported certbot flag --manual-public-ip-logging-ok with certbot 4.0.0 during taftauto renew dry-run.\n- No certificate, DNS, NPMplus route, or deploy state changed by the failed dry-run.\n\n## TAFTAUTO_CERT_RENEW_DRYRUN_RATE_LIMITED_20260702\n- Proof 672 is superseded/partial: it printed STATUS=OK even though certbot dry-run failed.\n- Failure reason: Let's Encrypt staging/service returned rateLimited / Service busy; retry later.\n- Cloudflare manual auth and cleanup hooks did run successfully.\n- Deploy hook was installed and manually invoked successfully before the dry-run.\n- Do not retry immediately.\n\n## HOMEPAGE_XENFORO_INFO_INLINE_PYTHON_SYNTAX_ERROR_20260702\n- Assistant used invalid inline Python syntax with def after semicolon while adding XenForo.info to Homepage External.\n- Failed before services.yaml write.\n\n## HOMEPAGE_USEFUL_ROUTER_NAME_MISMATCH_20260702\n- Previous apply looked for Russian Router card name, but active Homepage Useful group contains Router and Public Domain.\n- Failed before write. Fix uses YAML-aware edit with exact names Router/Public Domain.\n\n## HOMEPAGE_USEFUL_ROUTER_CARD_NOT_FOUND_20260702\n- Apply expected Useful -> Router in active services.yaml, but YAML parse did not find it. No write was performed.\n- Need verify active Homepage config path, docker mount, and any alternate services.yaml before next apply.\n\n## 20260702_CF_TOKEN_AUDIT_BROKEN_COMMAND\n- Ошибка: ассистент дал хрупкую команду аудита Cloudflare-токенов с вложенным heredoc/Python через ssh.\n- Факт: bash начал интерпретировать Python-код, появились syntax error и вторичный prompt >.\n- Правило: не давать nested heredoc/сложные кавычки через ssh; для длинной логики сначала класть скрипт отдельным файлом, затем запускать его.\n- Статус: аудит токенов не выполнен, инфраструктурные изменения не должны считаться применёнными.\n\n## LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE\n- Do not treat e3qxxx.netbird.selfhosted / 100.100.125.70 and e3qxxx-183-106.netbird.selfhosted / 100.100.183.106 as VPS egress nodes; they are user mobile peers.\n- Correct egress candidates are relay.pvepro.ru / relay.netbird.selfhosted / 100.100.19.1 for Moldova and mail.pvepro.ru / mail.netbird.selfhosted / 100.100.147.204 for USA.\n- Access Policy edge-vm -> relay/mail only proves peer reachability. It is not an internet exit route.\n- Required NetBird fix for CrowdSec CAPI was Network Routing exit route 0.0.0.0/0 distributed to edge-vm group, routing peer relay, masquerade/NAT enabled.\n- Before attempting CrowdSec CAPI, verify on edge-vm: ip route get 1.1.1.1 uses wt0, wg allowed-ips contains 0.0.0.0/0, and Cloudflare trace no longer shows home IP 95.84.154.183.\n\n## LESSON_20260702_HOMEPAGE_MONITORS_REPAIR_NOT_DELETE\n- Do not delete or disable Homepage siteMonitor fields to hide red badges.\n- Fix the underlying monitor path, DNS, proxy route, or local health-wrapper instead.\n- Do not touch Cloudflare Homepage card when operator says it is green and opens correctly.\n- Before changing Homepage, identify the live config host; in this state live Homepage is on core-apps [PRIVATE_IP], not edge-vm.\n\n## LESSON_20260702_KUMA_ADD_ONE_MONITOR_ONLY\n- When asked to fix one missing Uptime Kuma service, add only one monitor, then generate proposals separately.\n- Before direct Kuma DB mutation, stop the container and create a DB backup.\n- Verify DB integrity before starting Kuma again.\n- Do not touch Cloudflare when operator says it is green and opens correctly.\n\n## LESSON_20260702_DOCKGE_STALE_STACKS_NOT_CONTAINERS\n- Dockge inactive items after migration can be stale compose folders, not stopped containers.\n- First classify runtime projects across all Docker hosts before deleting or archiving anything.\n- For remote Dockge visibility, projects must be under /opt/stacks or bind-mounted there.\n- Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions.\n\n## STAGE4B_MASKED_SQL_QUERY_FAILURE_20260714\n- Previous Stage4B read-only audit returned empty observation-summary, metadata-key, trigger and function blocks because nested SSH/SQL quoting broke queries while stderr was hidden.\n- Empty blocks are query failures, not proof that metadata, triggers or functions are absent.\n- Future PostgreSQL audits must use ON_ERROR_STOP, visible stderr, independent query RC and quote-safe dollar literals.\n\n## STAGE4C_SCHEMA_MIGRATIONS_ID_ASSUMPTION_20260714\n- Stage4C read-only preflight incorrectly assumed schema_migrations has a column named id.\n- PostgreSQL returned column id does not exist and ON_ERROR_STOP correctly stopped all following SQL checks.\n- Production DB and application were not changed. Future migration audits must inspect information_schema first and query migration rows without assuming column names.\n\n## STAGE4C_PGDUMP_DEV_NULL_FSYNC_INVALID_PROBE_20260714\n- Stage4C custom-format pg_dump capability probe incorrectly used /dev/null as the output file.\n- pg_dump failed only because fsync on /dev/null returned Invalid argument.\n- Production database and application were not changed.\n- Future custom dump capability checks must use a real temporary regular file, validate it with pg_restore --list, and remove it afterward.\n\n## STAGE4C_REMOTE_PREP_THIRD_SUBSTEP_FAILED_20260714\n- Isolated migration dry-run did not start: REMOTE_PREP_RC=0|0|1.\n- Remote directory creation and SCP succeeded; the third ownership/mode preparation substep returned RC=1.\n- Production database and application were not changed.\n- Exact cause requires read-only residual directory and permission inspection before cleanup or retry.\n\n## STAGE4C_REMOTE_PREP_GLOB_AFTER_CHMOD_ROOT_CAUSE_20260714\n- Exact cause of REMOTE_PREP_RC=0|0|1: after the directory became postgres-owned mode 0700, the unprivileged debian shell could not traverse it and could not expand the wildcard used by chmod.\n- Candidate files and SHA256 verification were valid; no temporary database was created and production was unchanged.\n- Future preparation must use exact privileged paths or a verified shared group, never an unprivileged wildcard after restrictive chmod.\n\n## STAGE4C_MIGRATION003_FINGERPRINT_DOLLAR_QUOTE_FAILURE_20260714\n- Migration 003 isolated dry-run failed at fingerprint CHECK because the regex end anchor collided with the SQL dollar-quote delimiter and produced an unterminated dollar-quoted string.\n- The migration transaction rolled back, the temporary database was removed, and production remained unchanged.\n- Fingerprint validation must use length plus translate without a regex dollar anchor; corrected candidate must pass a fresh isolated dry-run before production consideration.\n\n## STAGE4C_SEAL_OUTER_RC_MASKING_20260714\n- Stage4C final seal correctly refused creation because overall health was WARN, returning internal RC=40.\n- The former wrapper then printed a second misleading COMMAND_RC=0 because top-level true masked the internal result.\n- Future commands must capture the guarded subshell through PIPESTATUS and print exactly one authoritative COMMAND_RC before the mandatory final true.\n- Production database, application and services were unchanged.\n\n## STAGE4C_DIAGNOSIS_EXPECTED_DERIVED_FILE_ON_VM180_20260714\n- Previous diagnosis incorrectly treated cluster-admin-restricted-probes.txt as a required source file on VM180.\n- The file is a central derived health artifact on pve01; VM180 supplies full-observer evidence instead.\n- This false expectation caused diagnostic RC=32 without any infrastructure failure or production change.\n\n## STAGE4C_NO_JOBS_RUNNING_TEXT_COUNTED_AS_JOB_20260714\n- Stage4C seal preflight incorrectly counted the systemctl informational text No jobs running as one active job.\n- The parser counted every nonempty output line instead of accepting only rows beginning with a numeric systemd Job ID.\n- No service was started and no production state changed during the blocked attempt.\n- Future job counts must match a numeric first field only.\n\n## STAGE4D_ADAPTER_SOURCE_AUDIT_ABORTED_BEFORE_FINAL_MARKERS_20260714\n- The evidence-only Stage4D adapter source audit returned RC=1 before emitting its VM180 completion and final audit markers.\n- The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults.\n- Production, database, application, services, timers, health and desired-state were unchanged.\n- Inspect the preserved runner stdout, stderr and result JSON before correcting or rerunning the task.\n\n## STAGE4D_LOCAL_EXPANSION_OF_REMOTE_Q_UNDER_NOUNSET_20260714\n- The first Stage4D adapter source audit embedded a large remote program inside a locally double-quoted SSH argument.\n- Local shell expansion reached the remote variable Q while nounset was enabled, causing Q: unbound variable before SSH execution.\n- VM180 and PostgreSQL audits did not start; production state was unchanged.\n- Remote shell and SQL programs must be separate syntax-checked files passed through SSH stdin.\n\n## STAGE4D_REMOTE_PYTHON311_ADAPTER_SELFTEST_RC1_20260714\n- Canonical adapter candidate passed local Python 3.13 self-tests with 48 envelopes but returned RC=1 on VM180 Python 3.11.2.\n- Remote upload and cleanup succeeded, and production database remained 0|0|OK.\n- The exact traceback and failing assertion must be captured by an isolated identical reproduction before changing the candidate.\n\n## STAGE4D_PYTHON311_FSTRING_COMPATIBILITY_AND_REPRO_VALIDATOR_20260714\n- Adapter candidate passed Python 3.13 but failed Python 3.11 parsing at Path(row[\"path\"]) inside a double-quoted f-string.\n- Exact fix is Python 3.11-compatible quoting: Path(row['path']).\n- The reproduction command captured the expected compile failure but incorrectly returned CAPTURE_BAD=1 because its validator required compile RC=0.\n- Production database remained 0|0|OK and desired-state remained clean.\n- Future adapter candidates must pass syntax and full self-tests on both pve01 and VM180 runtimes before acceptance.\n\n## STAGE4E_DESIGN_VALIDATOR_ABORTED_BEFORE_LOCAL_VALIDATION_MARKER_20260714\n- Restricted probe-agent design candidate compiled successfully but its local validator exited before producing LOCAL_VALIDATION_RC.\n- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults.\n- Active external probes were not executed and production state was unchanged.\n- Inspect the preserved validator traceback and exact assertion before modifying the candidate.\n\n## STAGE4E_VALIDATOR_SCANNED_PYC_AND_OWN_FORBIDDEN_LITERALS_20260714\n- Stage4E design validator recursively read every candidate file as UTF-8 after py_compile had created binary __pycache__ bytecode.\n- This caused UnicodeDecodeError before any design assertion failed.\n- The phase-wide forbidden-pattern scan also included validate_design.py itself, which necessarily contains the prohibited literals it verifies.\n- Validators must use an explicit text-file allowlist and scans must exclude validator implementation and binary artifacts.\n- Production, database, services, timers and desired-state were unchanged; active external probes were not executed.\n\n## STAGE4E_V2_VALIDATOR_FAILED_STATIC_COMPILE_20260714\n- Stage4E design candidate v2 failed static compilation before local design validation started.\n- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects.\n- The generated validator must be inspected at the exact SyntaxError line before another candidate is created.\n- Production, database, application, services, timers, health and desired-state were unchanged; active external probes were not executed.\n\n## STAGE4E_PIPESTATUS_LOST_AFTER_FIRST_ASSIGNMENT_20260714\n- VM180 validation copy used a pipeline and read PIPESTATUS in two separate assignments.\n- The first assignment reset PIPESTATUS, leaving COPY_RC empty and preventing remote validation and cleanup.\n- Retry must use explicit SCP operations without pipeline status parsing.\n- Production remained unchanged and active external probes were not executed.\n\n## STAGE4F_VALIDATOR_EXPECTED_HEALTH_REFRESH_FROM_NONWRITING_BACKUP_SCRIPT_20260714\n- The controlled backup service completed with Result=success and ExecMainStatus=0.\n- Validation incorrectly required cluster-admin-incident-engine-backup.txt to receive a fresh timestamp.\n- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.\n- Stage4F backup proof must use the service result, fresh latest JSON, produced artifacts and off-host/restore evidence instead.\n- The backup must not be rerun merely to satisfy the invalid health-file freshness assertion.\n\n## STRICT_RULE_20260714_CLOSE_TAILS_IMMEDIATELY\n- Failure class: переход к следующей задаче при наличии незакрытого хвоста.\n- Mandatory anti-regression: после ошибки запрещено продолжать другой scope до точной root cause, исправления, повторной проверки, очистки, proof и seal.\n- Closure gate: UNRESOLVED_TAIL_COUNT=0; BLOCKER_COUNT=0; TEMPORARY_ARTIFACT_COUNT=0; ROLLBACK_VERIFIED=YES; HEALTH=OK; PROOF_UPDATED=YES; REFERENCE_UPDATED=YES.\n- Если закрытие невозможно из-за внешней зависимости, опасной неоднозначности или обязательного решения пользователя, фиксировать BLOCKED/OPEN и не заявлять CLOSED.\n- Proof: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/report.txt\n\n## ERROR_20260714_DEPENDENCY_FACT_QUERY_GLOBAL_LIMIT\n- Symptom: dependency audit вернул только latest_collector_status.\n- Root cause: ORDER BY 1 LIMIT 1 в конце UNION ALL применился ко всему набору фактов.\n- Correction: latest collector status извлекается скалярным подзапросом внутри CTE facts; глобальный LIMIT отсутствует.\n- Anti-regression: multi-fact SQL audit обязан проверять точное ожидаемое число строк до извлечения значений.\n- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt\n- Status: CLOSED.\n\n## ERROR_20260714_COLLECTOR_PATCH_REQUIREMENT_PATTERN_MISMATCH\n- Symptom: MIGRATION_PLAN_COLLECTOR_PATCH_REQUIRED_COUNT=1 при ожидаемых двух подтверждениях.\n- Root cause: узкий шаблон учитывал collector_patch_required, но не collector_patch_required_after_stage4c.\n- Correction: оба поля проверяются отдельно, затем складываются как semantic requirement count.\n- Anti-regression: связанные JSON-контракты проверяются по собственным точным именам полей, а не одним унифицированным grep-шаблоном.\n- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt\n- Status: CLOSED; PRIOR_TAIL_UNRESOLVED_COUNT=0.\n\n## ERROR_20260714_PRIVILEGED_FILE_REDIRECTION_BEFORE_SUDO\n- Symptom: bash reported Permission denied while counting collector.py lines.\n- Root cause: `< collector.py` was opened by the unprivileged caller shell before sudo executed wc.\n- Correction: run sudo wc -l collector.py without caller-side input redirection.\n- Anti-regression: never combine sudo with `< protected-file`; pass the protected path as an argument to the privileged command.\n- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt\n- Status: CLOSED.\n\n## ERROR_20260714_COLLECTOR_RUN_COLUMN_ASSUMPTION\n- Symptom: SQL failed because completed_at did not exist.\n- Root cause: collector_runs columns were assumed instead of read from information_schema; actual names are finished_at and error_text.\n- Correction: assert required and forbidden column counts before querying recent runs.\n- Anti-regression: every schema-sensitive audit must discover and validate exact column names before the main query.\n- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n\n## ERROR_20260714_REMOTE_STDIN_ARGUMENT_SHIFT\n- Symptom: remote harness выполнил chmod для пути bash.\n- Root cause: дополнительный positional token bash был передан после SSH command и стал первым аргументом stdin-скрипта.\n- Correction: remote command вызывает bash -s -- с ровно тремя явно встроенными аргументами: candidate path, SHA256 и byte count.\n- Anti-regression: stdin-скрипт обязан валидировать $1 по разрешённому path pattern и затем подтверждать точные SHA256 и byte count; запрещено добавлять отдельный bash-token после remote command.\n- Closure proof: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/report.txt\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n\n## ERROR_20260714_OVERSIZED_COMMAND_PARSE_FAILURE\n- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`.\n- Root cause: чрезмерно длинная интерактивная команда была обрезана или повреждена при передаче до полного разбора Bash.\n- Impact: side-effect audit подтвердил production state 0|0|OK, неизменный collector, активный timer и отсутствие временных БД, каталогов и процессов.\n- Correction: монолитные команды запрещены; сложные действия выполняются через отдельный проверяемый task-скрипт и короткий launcher.\n- Anti-regression: interactive payload target <= 8000 bytes; syntax-check перед запуском; remote output сохраняется независимо от RC.\n- Closure proof: /var/lib/homelab-change-sets/operator-command-size-guard-v1/20260714T182653Z/report.txt\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n\n## ERROR_20260714_HOMELAB_ADMIN_HELP_RC_ASSUMPTION\n- Symptom: template audit завершился HOMELAB_ADMIN_HELP_FAILED.\n- Root cause: был ошибочно ожидаем RC=0 от неподдерживаемого аргумента --help.\n- Actual contract: неизвестный аргумент печатает usage и возвращает RC=64.\n- Correction: проверять version либо считать точную usage-строку вместе с RC=64 штатным результатом.\n- Anti-regression: перед проверкой CLI сначала фиксировать поддерживаемые команды; не предполагать GNU-style --help.\n- Closure proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v1/20260714T183338Z/report.txt\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n\n## ERROR_20260715_EXPECTED_NEGATIVE_RC_TRAP_CLASSIFICATION\n- Symptom: schema-v2 task v3 build stopped while intentionally testing the --invalid argument path.\n- Root cause: an expected RC64 was executed while the generic ERR trap remained active.\n- Correction: expected failures are evaluated only through an explicit if/else branch with immediate RC capture.\nANTI_REGRESSION_TOKEN=EXPECTED_RC64_INTERCEPTED_ERR_TRAP\n- Required pattern: if command; then rc=0; else rc=$?; fi, followed by a separate exact expected-RC assertion.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n\n## ERROR_20260715_DUPLICATE_MACHINE_MARKER_COUNT\n- Symptom: error-register candidate construction stopped with RC1 before applying the candidate.\n- Root cause: the same marker text appeared in both a heading and a descriptive line, while the guard expected one substring occurrence.\n- Correction: headings and prose do not contain machine-token values; validation counts only complete exact token lines.\nANTI_REGRESSION_TOKEN=DUPLICATE_MACHINE_TOKEN_IN_HEADING_AND_BODY\n- Required pattern: grep -Fxc against a complete machine line, never grep -Foc against an unrestricted substring.\n- Production impact: none.\n- Temporary artifacts: removed and verified.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T02:56:19Z\n\n## ERROR_20260715_OUTER_WRAPPER_SHELL_SYNTAX\n- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis.\n- Root cause: the outer wrapper contained an unbalanced compound shell construct before the closing subshell delimiter.\n- Impact: parsing failed before the start marker, guards and homelab-admin invocation; no task phase or infrastructure action executed.\n- Correction: use a linear wrapper with explicitly closed if/case blocks and avoid nested brace groups in conditional expressions.\nANTI_REGRESSION_TOKEN=OUTER_WRAPPER_UNBALANCED_SUBSHELL_SYNTAX\n- Anti-regression: after any parser-level failure, treat the attempted phase as not executed and verify package and run-state before retrying.\n- Production impact: none.\n- Task package impact: none.\n- Temporary artifacts: none.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T03:24:05Z\n\n## ERROR_20260715_INLINE_REMOTE_SHELL_IN_APPLY_PHASE\n- Symptom: homelab-admin lint rejected Stage4H task v6 with ERR-002-INLINE-REMOTE-SHELL.\n- Root cause: apply.sh invoked bash -s through an inline quoted SSH program and streamed the remote script through stdin.\n- Correction: copy the declared remote script to an isolated fixed path on VM180 and invoke that file directly with positional arguments.\nANTI_REGRESSION_TOKEN=INLINE_REMOTE_SHELL_REPLACED_BY_STAGED_REMOTE_SCRIPT_PATH\n- Anti-regression: phase scripts may perform transport orchestration, but remote shell program bodies must exist as separately declared, syntax-checked remote script files.\n- Remote stdout and stderr must be preserved before evaluating the remote return code.\n- Task v6 status: REJECTED_BY_LINT; never executed and never mutated.\n- Production database impact: none.\n- Live collector impact: none.\n- Temporary database and remote root count after rejection: zero.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T04:13:46Z\n\n## ERROR_20260715_OPTIONAL_RUN_DIRECTORY_FIND_UNDER_PIPEFAIL\n- Symptom: task-v7 builder stopped while counting formal runs for rejected task v6.\n- Root cause: find was executed against an optional directory that did not exist; under pipefail the assignment returned RC1.\n- Correction: test directory existence first and assign zero without invoking find when it is absent.\nANTI_REGRESSION_TOKEN=MISSING_OPTIONAL_RUN_DIRECTORY_TREATED_AS_ZERO\n- Anti-regression: optional paths must have an explicit existence branch before find under pipefail.\n- Production impact: none.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T04:44:06Z\n\n## ERROR_20260715_PG_CONSTRAINT_CONTYPE_CHAR_CONCAT\n- Symptom: isolated Stage4H acceptance failed during schema baseline capture.\n- Root cause: pg_constraint.contype uses the internal PostgreSQL char type and was concatenated without an explicit text cast.\n- Correction: use contype::text or CAST(contype AS text).\nANTI_REGRESSION_TOKEN=PG_CATALOG_CHAR_CONCAT_REQUIRES_EXPLICIT_TEXT_CAST\n- Machine rule: ERR-014-PG-CATALOG-CHAR-CONCAT.\n- Negative self-test: uncast expression rejected with RC64.\n- Positive self-test: explicit text cast accepted with RC0.\n- Task v7 mutated: no.\n- Production impact: none.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T06:27:06Z\n\n## ERROR_20260715_LIVE_COLLECTOR_VERIFIER_WRONG_PATH\n- Symptom: verified linter installation was rolled back after the final collector integrity guard returned UNKNOWN.\n- Root cause: verifier queried a nonexistent path instead of the canonical live collector path declared by the task acceptance script.\n- Canonical path: /opt/cluster-admin-incident-engine/collector.py.\n- Incorrect path: /opt/homelab-cluster-admin/cluster-admin-collector.py.\nANTI_REGRESSION_TOKEN=LIVE_COLLECTOR_VERIFIER_PATH_MUST_MATCH_TASK_CANONICAL_PATH\n- Correction: derive and guard the live collector path from the immutable task contract before executing the hash query.\n- Production impact: none; the canonical collector hash remained unchanged.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T06:27:06Z\n\nAUTOREFRESH_INLINE_FAILURE_20260721\nAttempts 048 and 049 did not appear in immutable history.\nRule: use validated local helper scripts and a simple scheduler entry; never embed the full collector in cron or nested SSH quoting.\n\nAUTOREFRESH_WRAPPER_PUBLICATION_FAILURE_20260721\nCommands 044-051 did not reach immutable history through homelab-chat-run.\nResolution: hourly snapshots use direct scp plus homelab-runtime-receive publication.\n\nAUTOREFRESH_052_NOT_PUBLISHED_20260721\nCONTEXT-AUTOREFRESH-ACTIVATE-052 did not appear in immutable history; direct publisher retry requires sanitization before transmission.\n\nIMMUTABLE_HISTORY_FALSE_NEGATIVE_20260721\nBACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative.\nRule: verify the exact COMMAND_ID result, wrapper publication status and commit SHA; never infer absence from a cached directory listing.\n\nBACKUP_UNSAFE_SOURCE_CAPTURE_071_PARSE_ERROR_20260721\nBACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.\nResolution: avoid nested quote layers; parse targets first and pass file paths as simple SSH arguments.\nDETAIL=SECTION_ERROR_REGISTER_END\nDETAIL=SECTION_STRUCTURED_ERRORS_INDEX_BEGIN\n{\n \"schema_version\": 1,\n \"channel\": \"homelab-runtime\",\n \"command_id\": \"ERRORS-INDEX-004\",\n \"status\": \"OK\",\n \"rc\": 0,\n \"host\": \"pve01\",\n \"mode\": \"read-only\",\n \"component\": \"error-ledger\",\n \"started_at_utc\": \"2026-07-21T07:22:22Z\",\n \"finished_at_utc\": \"2026-07-21T07:22:22Z\",\n \"reference_register_checked\": true,\n \"reference_sha256\": \"f6f7749fda946015116cdbbd754c7e7f20356b4a6cf4df330a1b0b8fdf59cbb2\",\n \"error_register_checked\": true,\n \"error_register_sha256\": \"ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc\",\n \"command_sha256\": \"a45b24ab6a161508f31ee8a6b954c045e74824b4d12adfef6f298cfac6111016\",\n \"duplicate_failed_command_blocked\": false,\n \"block_reason\": null,\n \"execution_started\": true,\n \"changes_made\": false,\n \"sanitized\": true,\n \"secrets_included\": false,\n \"private_addresses_included\": false,\n \"raw_evidence_retained_locally\": true,\n \"raw_evidence_sha256\": \"655ec287a9095e799677cb7f87b08d4119cbdf9027d7097fa610da5a022ab30b\",\n \"sanitized_output_sha256\": \"655ec287a9095e799677cb7f87b08d4119cbdf9027d7097fa610da5a022ab30b\",\n \"output_truncated_in_json\": false,\n \"full_sanitized_output_url\": \"https://git.gram1.ru/.well-known/homelab-runtime/latest.txt\",\n \"output\": \"{\\n \\\"schema_version\\\": 1,\\n \\\"status\\\": \\\"READY\\\",\\n \\\"generated_at_utc\\\": \\\"2026-07-21T07:22:22.821762Z\\\",\\n \\\"source\\\": {\\n \\\"path\\\": \\\"/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\\\",\\n \\\"sha256\\\": \\\"ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc\\\",\\n \\\"line_count\\\": 934,\\n \\\"sanitized\\\": true\\n },\\n \\\"summary\\\": {\\n \\\"entry_count\\\": 154,\\n \\\"rule_count\\\": 90,\\n \\\"duplicate_entry_ids\\\": [],\\n \\\"duplicate_entry_signatures\\\": [\\n \\\"9cd06e064f5a86be9626c4f9a2a819b8d0e71b03309c705fe0e34e1e59f81b73\\\"\\n ],\\n \\\"duplicate_rule_signatures\\\": [\\n \\\"b936ae30ffb935118585b5a133d9c32643946a3ba5034b85f42bae000518d49a\\\"\\n ]\\n },\\n \\\"entries\\\": [\\n {\\n \\\"id\\\": \\\"ERR-H-L1-HOMELAB-ASSISTANT-ERROR-REGISTER\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 1,\\n \\\"source_line\\\": 1,\\n \\\"title\\\": \\\"HOMELAB ASSISTANT ERROR REGISTER\\\",\\n \\\"summary\\\": \\\"Назначение: перед каждой следующей командой сверяться с этим файлом.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"cd4b4d42c26702b03e3e81a620c46dbe3d498f3f2233215024a5c2a49cc1dbfc\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-1-L6\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 6,\\n \\\"title\\\": \\\"Повторно дал слишком большой интерактивный paste в shell.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"f732778534f22b73e31dead215c9c7078f48a552a5341ae3830eea618885e8a7\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-2-L7\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 7,\\n \\\"title\\\": \\\"Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"b13c8946e274927bd041960c76208b6f62e22d46f07233388878c1f47afc426a\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-3-L8\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 8,\\n \\\"title\\\": \\\"Дал генератор справочника прямо в терминал вместо безопасного маленького шага.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"8de70b8d263233557a33a065ee49004ef01e328de0d736eb9e328930119458be\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-4-L9\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 9,\\n \\\"title\\\": \\\"Нарушил своё же правило: не давать длинные вложенные команды с кавычками.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"d1bf3e42845e6c9d25511814c1da9b21cd8001474f17fd60240194f812f06fe0\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L5-КРИТИЧЕСКИЕ-ОШИБКИ-АССИСТЕНТА\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 5,\\n \\\"title\\\": \\\"Критические ошибки ассистента\\\",\\n \\\"summary\\\": \\\"1. Повторно дал слишком большой интерактивный paste в shell. 2. Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт. 3. Дал генератор справочника прямо в терминал вместо безопасного маленького шага. 4. Нарушил своё же правило: не давать длинные вложенные команды с кавычками.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"0ba065b54f515fe668143ec87d006ea8f27639e6529fe46367ac9acd6f2e28d6\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L11-ЖЁСТКИЕ-ПРАВИЛА-ПЕРЕД-КАЖДОЙ-КОМАНДОЙ\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 11,\\n \\\"title\\\": \\\"Жёсткие правила перед каждой командой\\\",\\n \\\"summary\\\": \\\"CHECK-1: команда не должна быть большим paste. CHECK-2: команда не должна содержать большой here-doc. CHECK-3: команда не должна смешивать Markdown, backticks и shell-логику. CHECK-4: команда не должна иметь вложенный ssh с несколькими уровнями кавычек. CHECK-5: команда не должна печатать секреты. CHECK-6: если создаётся файл, сначала маленький безопасный шаг, потом проверка. CHECK-7: если команда длиннее 8 строк, её нельзя давать в интерактивный shell. CHECK-8: для edge-vm использовать debian@[PRIVATE_IP] и sudo.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"7ac085a9df3e592d1d7d6686ca4980278110a07e296ddc0253cedb075cc08d84\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L23-ТЕКУЩИЕ-ВАЖНЫЕ-ФАКТЫ\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 23,\\n \\\"title\\\": \\\"Текущие важные факты\\\",\\n \\\"summary\\\": \\\"Internal network: [PRIVATE_IP]/24. Migration config: migration: secure,network=[PRIVATE_IP]/24. Corosync remains on [PRIVATE_IP]/12/13. VM160 forum-prod is not in Proxmox nightly backup. VM130 edge-vm has secondary disk backup=0 risk. 05_edge_compose_safe.tgz quarantined.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"ecd7f734df435c41c5f01c9fd7f9371a4012c6019493072741c45c6eaf3ed383\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-11-L35\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 35,\\n \\\"title\\\": \\\"Ошибка: считать offhost OK после failed rsync.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"aa8a2a5cc3228d73d2feadd69fa2fb1d0f76d020280f69921ac5c00dcfc1f9fd\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-12-L40\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 40,\\n \\\"title\\\": \\\"Ошибка: широкий secret-поиск по /opt/stacks дал шум.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"da33bb4eaab4ca4827c1e122c4292fa4181a7607db77ebce5d185b867bfe0ac3\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-13-L45\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 45,\\n \\\"title\\\": \\\"Ошибка: SQL с одинарными кавычками внутри одинарной SSH-команды ломается.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"888cbc312835f6303e74db4c5ed2dcaef1e552e657f83ca6f3f82834eb06c23e\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-14-L50\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 50,\\n \\\"title\\\": \\\"Ошибка: путать контекст входа и узел выполнения.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"97eec6b76b51ffbc78105ce4e450846c5ff1c85dfbcdfeb3e6c687addd6a1391\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-15-L57\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 57,\\n \\\"title\\\": \\\"Ошибка: повторно нарушено правило №13 после его добавления.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"95160c6fda4d8bc77fc542236363b182ba66a7314a8032379b033a0feba5e3b2\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-16-L63\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 63,\\n \\\"title\\\": \\\"Основополагающее правило: перед каждой командой ассистент обязан явно написать: Сверка с файлом ошибок: CHECK пройден.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"254db5ca3a2611269de7af81ececd43035134e01a8e629c34434913c49eeeb41\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-19-L68\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 68,\\n \\\"title\\\": \\\"Уточнение формата: не писать отдельную строку сверки перед каждой командой в пакете.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"a77bda86805176e2dc721961454b31a5bff15fe924bd43496475c5816a8f9650\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-20-L73\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 73,\\n \\\"title\\\": \\\"Ошибка: широкий XenForo SMTP-поиск по src дал шум исходников.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"bed26484549b8d47c75ee6150c206503c05074cc8a5b25dfde0735596aaf956c\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-21-L77\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 77,\\n \\\"title\\\": \\\"Ошибка: nested PHP php -r дал Parse error на forum-prod.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"1198d8f0576c5dc8e3883ac7d7e423c8391e946c73bf5ac3cf6063278a68a5af\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-22-L82\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 82,\\n \\\"title\\\": \\\"Ошибка: самодельный base64 PHP для SMTP auth сломан.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c629522366fb27522bec6dd9b9b24f532bbc9c3372fbf4d706673bf283bdf498\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-23-L87\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 87,\\n \\\"title\\\": \\\"Ошибка: MariaDB не поддержала cast('true' as json) в JSON_SET.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"4faccf6aa3103144040fd4efb62dd82e23d9f28ecfba72502293f6e2f060b819\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-24-L92\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 92,\\n \\\"title\\\": \\\"Ошибка: exit 1 в interactive-check закрыл SSH-сессию.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c522fbe5bc7aadfc904f1bde7328952b554884941afb3558e60cfe6a4ed378cd\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-25-L97\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 97,\\n \\\"title\\\": \\\"Ошибка анализа: не надо слепо включать smtpSsl=true при smtpPort=587.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"17417cdf64e208886d887fd449a612090a81aa5a8f47944f17993821ad5e8235\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-26-L102\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 102,\\n \\\"title\\\": \\\"Ошибка: openssl -crlf вместе с ручным CRLF ломает SMTP AUTH checker.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"3f83176fadcfeb2318a3e3fc75c13e3944d3fef3c0aa7aef40db3736b27e794a\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-27-L108\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 108,\\n \\\"title\\\": \\\"Ошибка: Python heredoc внутри ssh сломал not_ok диагностику.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"437f761644210f81c8ca3333410a58585c7b74c54a983b9d84159d98c49b4847\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-28-L112\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 112,\\n \\\"title\\\": \\\"Ошибка: docker compose ls --format json | wc -l дал ложный COMPOSE_PROJECT_COUNT=1.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"752e0913f85275527a448868cce24aa85cf06949ea82e066e4dc0fc2e857d1ad\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-29-L116\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 116,\\n \\\"title\\\": \\\"Ошибка: monitoring compact status искал неверные имена health-файлов.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"61df0580bfb53db159bb0c3fced23235437e823a877822e0b8aebb7518453d4d\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-30-L121\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 121,\\n \\\"title\\\": \\\"Ошибка/инцидент: Prometheus API HTTP_CODE=000 после monitoring reference.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c0dac19354e77153d3f61d53659a8e694c57308bc564e03c833cb68831e88031\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-31-L125\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 125,\\n \\\"title\\\": \\\"Строгое правило: перед каждой командой сверяться с файлом ошибок и справочником.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"fa4e49944e2caf8ba4f26a6064bfb465c035e8bc3184244c0314812e81bd6df5\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-32-L130\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 130,\\n \\\"title\\\": \\\"Ошибка: 153_PROMETHEUS_TARGETS_SETTLED_PROOF содержит SyntaxError из-за вложенного Python внутри SSH.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"13b3234027d04af4194a5352dda2a5bd21ab955751eb161b953637347da8febf\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-34-L135\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 135,\\n \\\"title\\\": \\\"Ошибка операционного ввода: полный терминальный лог MobaXterm попал в shell.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"9a7930e756982125ac45f47a4adf7dfdb10906bd950db4ddae3659021e01bc64\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-33-L140\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 140,\\n \\\"title\\\": \\\"Security finding: root authorized_keys на PVE-нодах имел права 777.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"44a7ef2e7d0428388c9b30d12ecfad460caaca457b5d283ebf00ae87c1b2119c\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-35-L144\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 144,\\n \\\"title\\\": \\\"Ошибка проверки: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED не доказал исправление прав.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"336bfad03f4600ddc0d8996bdbbebeae51cfa0c1e2281ae1a838b8b4d8bc7dfd\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-36-L149\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 149,\\n \\\"title\\\": \\\"Quality check: Storage block needs integrity and pve03 capacity coverage review.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"a11f29a5da892017620722200a35bdde4c0fe5aa591de4331bfbee0851e38266\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-37-L154\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 154,\\n \\\"title\\\": \\\"Coverage gap: pve03_staging missing from disk-space health coverage.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"d2fded36e2e512289b08c31b7fc6872e4efbf61c06e2840e5a2dfef9a8e989f8\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-38-L158\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 158,\\n \\\"title\\\": \\\"Quality check: Service Dependency Map block needs integrity review.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"79cffb917505e112ac9507770da4068e145c75d2278d50d1034bb594b1da5062\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-39-L162\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 162,\\n \\\"title\\\": \\\"Ошибка проверки: post deep audit BAD_MARKER scan caught prose false positive.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"6dc6e561ff4c49c39a8d2f7fdac4fbfd9d2b6295212185791fb9091f644a2824\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-40-L166\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 166,\\n \\\"title\\\": \\\"Ошибка/вывод: VM150 Mail cloud single-file upload failed with 413.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"d7e096eac4cbc2b602601af11ce42bb716d28142a278203aefb65699b5fd4b9c\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-41-L170\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 170,\\n \\\"title\\\": \\\"Ошибка: post-backup-pass-slo-reconciliation checked pve01 router health on edge-vm.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"44ceb289e0963e0e97c6e8019e93120ad638f5648e8596b987e7d413eaeb59aa\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L31-ПРАВИЛО-ДЛЯ-СПРАВОЧНИКА\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 31,\\n \\\"title\\\": \\\"Правило для справочника\\\",\\n \\\"summary\\\": \\\"Не генерировать большой справочник через интерактивную вставку. Следующий справочник делать только маленькими append-блоками или через файл, созданный вне интерактивного shell. 11. Ошибка: считать offhost OK после failed rsync. Если rsync/scp упал, нельзя проверять latest offhost-файл без сверки имени. Проверка должна подтвердить именно новый архив, например дату 2026_06_30-00_20_59. Старый OFFHOST_ZSTD_OK не закрывает новый backup. 12. Ошибка: широкий secret-поиск по /opt/stacks дал шум. Не искать Cloudflare/SMTP сразу по /opt/stacks, MkDocs site, backups и HTML-отчётам.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"2c69d96d9f906cc88ab3158941ba041ca57ddee1e03554fad588eea96041930a\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L174-ASSISTANT-COMMAND-BATCHING-RULE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 174,\\n \\\"title\\\": \\\"ASSISTANT_COMMAND_BATCHING_RULE_20260630\\\",\\n \\\"summary\\\": \\\"- Rule: combine related infrastructure checks, proof generation, reference updates and validation scans into one safe batch command when practical. - Do not split work into many small commands when the operations are low-risk, read-only, or naturally part of one evidence block. - Still keep mandatory preflight checks, short bounded commands, no large paste, no here-docs, no interactive shell traps, and no sensitive values in output. - Split into smaller commands only when safety, rollback, quoting risk, or troubleshooting clarity requires it.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"73ef09d525c334b9874616d5bafdf17d4898a815d35e736e33df0b83790f6539\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L180-HOME-PORTAL-BASE64-APPLY-FAILURE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 180,\\n \\\"title\\\": \\\"HOME_PORTAL_BASE64_APPLY_FAILURE_20260630\\\",\\n \\\"summary\\\": \\\"- Prior home-portal apply attempt 312 failed because a large base64 Python payload decoded as invalid UTF-8. - Rule: avoid large opaque base64 script payloads for homelab changes; use transparent bounded shell/perl/python commands instead.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"4964d3ff556bb377d7c9d0cb4d487afabc0d8ac69c7fb2a5fccb5ff077a8a478\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L184-HOMELAB-COMMAND-SAFETY-HARDENING-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 184,\\n \\\"title\\\": \\\"HOMELAB_COMMAND_SAFETY_HARDENING_20260630\\\",\\n \\\"summary\\\": \\\"- Rule: do not use large opaque payloads, generated base64 scripts, or hidden multi-line script blobs for homelab changes. - Use transparent bounded shell/perl/python commands, or create a temporary readable script and validate it before execution. - Any script used for changes must pass a syntax/parse check before it is allowed to modify files or restart services. - Any Traceback, UnicodeDecodeError, SyntaxError, Permission denied, command-not-found, nonzero critical RC, or failed write marker must force REVIEW or FAILED, never OK. - Success requires both runtime health and content-specific post-checks: expected files changed, expected cards exist, duplicates absent, service/container alive, and no new critical log pattern. - HTTP 200 alone is not a success condition for configuration changes. - If an apply step fails, the next command must explicitly record the failed attempt and then run a corrected transparent apply command.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"463e648fd9e6d671debe81d12b455563460bc669983e083ef5f0f3b4df2f208b\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L193-HOME-PORTAL-API-ERROR-CLOSURE-RULE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 193,\\n \\\"title\\\": \\\"HOME_PORTAL_API_ERROR_CLOSURE_RULE_20260630\\\",\\n \\\"summary\\\": \\\"- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200. - If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed. - External informational widgets such as weather/Open-Meteo must not block the service launcher portal.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"1a011c6be8607fdacb9571baa017241dcb441a0025f985ecce3d0937a4f7e3b5\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L198-HOME-PORTAL-COMPLEX-QUOTING-FAILURE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 198,\\n \\\"title\\\": \\\"HOME_PORTAL_COMPLEX_QUOTING_FAILURE_20260630\\\",\\n \\\"summary\\\": \\\"- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('. - Cause: command was too complex and fragile due to nested shell/perl/python quoting. - Rule: do not use deeply nested one-liners for config edits; first run short state checks, then apply small transparent edits.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"862820180d4dbc46a845af7c9ec8847aeb21cda3fd7634c0d2afd9b3d14d5d8b\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L203-HOME-PORTAL-ACTIVE-CONFIG-SCOPE-RULE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 203,\\n \\\"title\\\": \\\"HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RULE_20260630\\\",\\n \\\"summary\\\": \\\"- Rule: Homepage cleanup validation must check active config files only, not logs or backup files. - Active files are services.yaml, bookmarks.yaml, widgets.yaml, settings.yaml and docker.yaml under /opt/stacks/homepage/config. - Matches in /logs/ or *.bak* are historical evidence only and must not make cleanup REVIEW unless the active config still contains the target.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"6bfd4c200f43d77ecd18a1c61c377b4c0588a028fb0ca6d565b2361d16a74967\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L208-HOME-PORTAL-LINK-VALIDATION-NPMPLUS-DEFAULT-RULE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 208,\\n \\\"title\\\": \\\"HOME_PORTAL_LINK_VALIDATION_NPMPLUS_DEFAULT_RULE_20260630\\\",\\n \\\"summary\\\": \\\"- Rule: Homepage link validation must not treat HTTP 200 as success if the body is the NPMPlus default “Congratulations” / host-not-set-up page. - Green siteMonitor dots can be false positives when NPMPlus returns its default vhost page. - A portal card is openable only when DNS, reverse-proxy host mapping, and target content are all valid.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"90363c457faef01bbe65d3e246073788fb426212371de346152c7776d246413b\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L213-ROUTER-CLI-STDIN-APPLY-FAILURE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 213,\\n \\\"title\\\": \\\"ROUTER_CLI_STDIN_APPLY_FAILURE_20260630\\\",\\n \\\"summary\\\": \\\"- Context: attempted Netcraze router ACL apply through SSH stdin/multiline for Homepage Moscow Router monitor fix. - Evidence: 391_MOSCOW_ROUTER_EDGE_VM_WEBADMIN_ACL_APPLY_PROOF.txt. - Failure: STDIN_SHOW_OK=no, APPLY_SKIPPED=stdin_show_failed, CONFIG_HAS_EDGE_5080=no, CONFIG_HAS_EDGE_5083=no. - Rule: do not apply Netcraze config through stdin/multiline SSH; prove CLI input mode first.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"7a151b61221247f7e5ecc6488283190d82b489caa9309675c08baded25d7fc6f\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L219-ROUTER-CLI-PROBE-STDIN-CONSUMPTION-ERROR-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 219,\\n \\\"title\\\": \\\"ROUTER_CLI_PROBE_STDIN_CONSUMPTION_ERROR_20260630\\\",\\n \\\"summary\\\": \\\"- Context: ACL syntax read-only probe loop executed only one command because ssh consumed the loop stdin. - Evidence: 393_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_PROOF.txt showed COMMAND_COUNT=1. - Fix pattern: use ssh -n or redirect SSH stdin away from the command-list loop for all future SSH-in-loop probes.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c801838e0f61bbd4f111bc30b21d530e69124b4448859e0e5f28a93cf0f01fec\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L224-ROUTERBACKUP-READONLY-CLI-PERMISSION-LIMIT-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 224,\\n \\\"title\\\": \\\"ROUTERBACKUP_READONLY_CLI_PERMISSION_LIMIT_20260630\\\",\\n \\\"summary\\\": \\\"- Context: routerbackup user can run show running-config but cannot execute configuration commands such as access-list. - Evidence: 394_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_FIXED_PROOF.txt. - Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli]. - Rule: do not attempt ACL/config changes as routerbackup; use admin/operator router access or router Web UI for write changes.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"1d8075c128dbd4b8f34929055e7c4b7c8cfda8f44f0b7874f04afbcc745d5d82\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L230-PROOF-SUMMARY-EXTRACTION-BLANK-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 230,\\n \\\"title\\\": \\\"PROOF_SUMMARY_EXTRACTION_BLANK_20260630\\\",\\n \\\"summary\\\": \\\"- Context: proof 395 tried to read COMMAND_COUNT/ZERO_RC_COUNT from proof files 393/394, but those summary fields were printed to console and were not persisted inside the proof files. - Evidence: 395_ROUTER_CLI_ERROR_REGISTER_UPDATE_PROOF.txt has blank P393_COMMAND_COUNT/P394_COMMAND_COUNT/P394_ZERO_RC_COUNT fields. - Rule: when creating follow-up proof summaries, derive counts from persisted proof body lines such as CMD_INDEX and CMD_RC, or write summary fields into the proof file itself.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"1fd6a207e921a706e6d8d3e80984d0436e67192bb56d03c91d20f6b33eba15cd\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L235-ROUTER-ACL-UI-RULE-DELETE-AND-RESTORE-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 235,\\n \\\"title\\\": \\\"ROUTER_ACL_UI_RULE_DELETE_AND_RESTORE_20260701\\\",\\n \\\"summary\\\": \\\"- Context: while moving Moscow Router monitor rules to _WEBADMIN_Bridge1, two _WEBADMIN_Bridge0 Nextcloud Talk TURN rules were accidentally removed. - Deleted/restored rules: Home [PRIVATE_IP]/24 to Nextcloud [PRIVATE_IP] TCP/UDP 3478. - Verification proof: 399_ROUTER_ACL_RESTORE_AFTER_UI_PROOF.txt. - Rule: after manual router Web UI ACL edits, compare _WEBADMIN_Bridge0, _WEBADMIN_Bridge1 and _WEBADMIN_GigabitEthernet1 before continuing.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"973a47bd8e8f388786157b6d50b53de9fc96b31e582a9645f3cf4189b74ac4fe\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L241-ROUTER-MONITOR-NODE-LOOKUP-PROBE-BUG-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 241,\\n \\\"title\\\": \\\"ROUTER_MONITOR_NODE_LOOKUP_PROBE_BUG_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 400 tested Moscow router HTTPS monitor from Homepage container with custom Node lookup callback. - Failure: NODE_HTTPS_RESOLVE_ERROR=ERR_INVALID_IP_ADDRESS and NODE_HTTPS_RESOLVE_INSECURE_ERROR=ERR_INVALID_IP_ADDRESS. - Impact: Node HTTPS resolve subtest is invalid; curl --resolve results remain valid. - Rule: do not rely on custom Node lookup callback until separately tested; prefer curl --resolve or simple direct Node HTTP checks.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"97151454d5ffda69d45b802ed15dc82057b02632e141b3729fa8ad47a959c556\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L247-ROUTER-HTTP-SERVICE-FORBIDDEN-FROM-EDGE-VM-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 247,\\n \\\"title\\\": \\\"ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701\\\",\\n \\\"summary\\\": \\\"- Context: Moscow Router Homepage monitor after ACL fix. - Evidence: proofs 399, 400, 401. - Observed: edge-vm TCP to [PRIVATE_IP]:5080/5083 is OK, but router HTTP/HTTPS returns 403 for all tested paths. - Impact: Homepage siteMonitor cannot be made green via direct router HTTP URL until router web service allows the Bridge1/Proxmox/protected segment or an explicit safe monitor endpoint is used. - Rule: do not keep changing ACL for this symptom; ACL is already correct. Fix service access scope or use a dedicated monitor endpoint.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"b0de3053a46b06841cbaf6ea5945be49ad8d451c7c8453d899d01a9f520cc384\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L254-PY-COMPILE-PYC-PERMISSION-ERROR-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 254,\\n \\\"title\\\": \\\"PY_COMPILE_PYC_PERMISSION_ERROR_20260701\\\",\\n \\\"summary\\\": \\\"- Context: installing edge-vm Moscow router health endpoint. - Mistake: assistant used `python3 -m py_compile` on a script installed under /usr/local/sbin as root. - Actual impact: non-root syntax check attempted to create /usr/local/sbin/__pycache__ and failed with Permission denied. - Recovery: validate syntax with `ast.parse` instead of py_compile, because it does not write .pyc files. - Rule: do not use py_compile against root-owned system paths from an unprivileged user.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"683a661513a8d46a89bf6492c8f12564f6f9fe262ce16b0f6487f63c0f2666f0\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L261-ROUTER-HOME-IP-DOCKER-ROUTE-CONFLICT-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 261,\\n \\\"title\\\": \\\"ROUTER_HOME_IP_DOCKER_ROUTE_CONFLICT_20260701\\\",\\n \\\"summary\\\": \\\"- Context: Moscow Router Homepage siteMonitor attempt using http://[PRIVATE_IP]:5080. - Evidence: proof 405 showed route to [PRIVATE_IP] via Docker bridge and Homepage container ECONNRESET. - Impact: do not use [PRIVATE_IP] directly as Homepage siteMonitor from edge-vm. - Mitigation: use dedicated edge-vm health endpoint that checks router TCP [PRIVATE_IP]:5080 and returns HTTP 200/503 for Homepage.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"5daeb3cf0b7d3bf64ddf6ca87e958f73d0715672617706dee233fd739fb08c2c\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L267-HOMEPAGE-ROUTER-MOSCOW-YAML-TITLE-SHAPE-MISMATCH-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 267,\\n \\\"title\\\": \\\"HOMEPAGE_ROUTER_MOSCOW_YAML_TITLE_SHAPE_MISMATCH_20260701\\\",\\n \\\"summary\\\": \\\"- Context: applying Moscow Router Homepage siteMonitor health endpoint. - Mistake: assistant apply script matched only property-style cards with `title`, but active Homepage YAML can use service-name-key style like `- Service Name:`. - Evidence: proof 407 returned YAML_CHANGE_COUNT=0 and APPLY_SKIPPED=unexpected_match_count while health endpoint proof 406 was OK. - Recovery: use YAML-aware updater that handles both property-style and service-name-key style cards.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"d7ed6a0ecfb19584319e97626e76661541c675e9125e4bf4ff0a407c9718cdd0\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L273-HOMEPAGE-ROUTER-MOSCOW-APPLY-SCRIPT-SYNTAX-ERROR-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 273,\\n \\\"title\\\": \\\"HOMEPAGE_ROUTER_MOSCOW_APPLY_SCRIPT_SYNTAX_ERROR_20260701\\\",\\n \\\"summary\\\": \\\"- Context: applying Moscow Router Homepage health endpoint. - Mistake: assistant generated Python script with invalid f-string escaping. - Evidence: proof 408 showed SyntaxError in /tmp/homepage-router-moscow-apply-fixed.py. - Actual impact: YAML was not changed, so Homepage green dot could not appear. - Recovery: stop applying blind updater; first discover exact active YAML shape, then apply minimal YAML-aware edit. - Rule: validate generated Python syntax locally before scp/remote execution; avoid f-strings in generated maintenance scripts.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"057b7e036316f6f240cac06b9f46e98d8399df689f82b903cc9ef630afe09f94\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L281-FORUM-PROD-VM160-FIRST-BOOT-SSH255-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 281,\\n \\\"title\\\": \\\"FORUM_PROD_VM160_FIRST_BOOT_SSH255_20260701\\\",\\n \\\"summary\\\": \\\"- Context: clean rebuilt VM160 first boot. - Symptom: PING_OK=yes and TCP22_OK=yes, but SSH_RC=255. - Rule: do not proceed with OS baseline until SSH failure is diagnosed; likely old known_hosts key or cloud-init/root-key issue.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"5e0359dc6b12e39f533e9060876f5c5f830b43cf2dbbe0441d6876fee5f410a3\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L286-FORUM-PROD-VM160-NESTED-HOSTNAME-PROOF-QUOTING-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 286,\\n \\\"title\\\": \\\"FORUM_PROD_VM160_NESTED_HOSTNAME_PROOF_QUOTING_20260701\\\",\\n \\\"summary\\\": \\\"- Context: VM160 first SSH proof after rebuild. - Issue: command substitution $(hostname) inside nested ssh was expanded on pve02 before entering VM160. - Impact: VM_HOSTNAME=pve02 in proof 426 is not valid VM identity evidence. - Rule: for nested SSH identity checks, run literal hostname commands without local command substitution.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c7174e4703945abd82004fea3fa9da0ff3efa1f2a89e6d2315cacf9c27b6d761\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L292-FORUM-PROD-VM160-SWAP-PROOF-QUOTING-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 292,\\n \\\"title\\\": \\\"FORUM_PROD_VM160_SWAP_PROOF_QUOTING_20260701\\\",\\n \\\"summary\\\": \\\"- Context: VM160 swapfile proof 429. - Issue: nested command substitution expanded on pve02, producing invalid SWAPFILE/FSTAB proof lines. - Impact: proof 429 is not valid closure evidence even though swap was active. - Rule: avoid nested $(...) in VM SSH proofs; use literal remote commands and clean proof.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"3dc2ea2b0a54adb8886979de4e01cb5f23704446f5b2198af9bdacc951359e4c\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L298-FORUM-UPLOAD-ARCHIVES-VM-VERIFY-QUOTING-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 298,\\n \\\"title\\\": \\\"FORUM_UPLOAD_ARCHIVES_VM_VERIFY_QUOTING_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 446 copy/check archives inside VM160. - Issue: nested SSH quoting expanded shell variables incorrectly, producing gzip checks against empty .gz and blank TAR_TOP lines. - Impact: SCP_TO_VM_RC=0 and file size list are valid, but GZIP/TAR proof lines in 446 are invalid. - Rule: verify VM archive integrity locally inside forum-prod, not through nested quoted loops.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"0e4a3a2f7c179f46c235e1ffa176b57485c88b4f7e7314bb89b6acb6683fe448\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L304-FORUM-PROD-BULK-IMPORT-PHP85-EMPTY-FRONTEND-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 304,\\n \\\"title\\\": \\\"FORUM_PROD_BULK_IMPORT_PHP85_EMPTY_FRONTEND_20260701\\\",\\n \\\"summary\\\": \\\"- Context: rebuilt VM160, five XenForo forums imported in bulk on PHP 8.5.7. - Symptom: all five forums returned HTTP 200 with empty body; XenForo logged Template public:PAGE_CONTAINER/forum_list is unknown. - Additional failed approach: proof 480 ran xf-dev:rebuild-caches on CodeVipe and returned RC=1, not a valid fix. - Decision: stop bulk repair, treat current VM160 as lab, find a working CodeVipe recipe first, then destroy/rebuild cleanly for all five forums. - Rule: no more mass rebuild/repair commands across all forums until single-forum CodeVipe recipe is proven.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"ea3c74648eece7c2a94559d8f7385ea4a126b399e3a161c5113e69a7b8d05f93\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L311-XENFORO-REQUIREMENTS-DIRECT-DOWNLOAD-403-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 311,\\n \\\"title\\\": \\\"XENFORO_REQUIREMENTS_DIRECT_DOWNLOAD_403_20260701\\\",\\n \\\"summary\\\": \\\"- Context: Tried to download XenForo requirements ZIP directly from xenforo.com inside forum-prod. - Issue: curl returned HTTP 403; unzip then failed because the ZIP was not downloaded. - Evidence: proof 491 showed DOWNLOAD_RC=22, ZIP_LIST_RC=9, UNZIP_RC=9. - Impact: proof 491 is not a valid server compatibility test. - Rule: use operator-uploaded xenforo23-requirements-test.zip from /root/forum-upload instead of relying on direct curl download.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"13164e6750f679553f0bb7cb13f4b895f345dc3f76c978349dfd30b93294206b\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L318-PVE01-UNZIP-MISSING-FOR-XF-REQUIREMENTS-VERIFY-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 318,\\n \\\"title\\\": \\\"PVE01_UNZIP_MISSING_FOR_XF_REQUIREMENTS_VERIFY_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 492 tried to inspect uploaded xenforo23-requirements-test.zip on pve01. - Issue: pve01 does not have unzip installed, so ZIP_TEST_RC=127 was not a ZIP integrity result. - Impact: proof 492 confirmed file presence only, not archive validity. - Rule: verify ZIP with Python zipfile or inside forum-prod rather than installing unzip on Proxmox just for inspection.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"341e814b6c0aab80d0bde631353118c8c04593d918a2ed7b60551d5bc3043148\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L324-FRESH5-DEPLOY-SUCCESS-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 324,\\n \\\"title\\\": \\\"FRESH5_DEPLOY_SUCCESS_20260701\\\",\\n \\\"summary\\\": \\\"- Context: VM160 was destroyed/recreated, then five XenForo forums were deployed from fresh ZIP backups. - Result: proof 513 confirms all five forums locally healthy. - Note: proof 511 initially failed dsmods path detection; proof 512 fixed dsmods by locating public_html via src/XF.php marker. - Rule: future restores should preserve internal_data/code_cache or rebuild compiled template cache before smoke testing.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"f46e9779cb65eebc42015f7042a4f55f0a8aa1e59e647eb29d728e48ed6069b3\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L330-NPMPLUS-SQLITE-PASTE-FAILURE-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 330,\\n \\\"title\\\": \\\"NPMPLUS_SQLITE_PASTE_FAILURE_20260701\\\",\\n \\\"summary\\\": \\\"- Context: NPMplus SQLite inspection command was too complex and a Python fragment was pasted into bash. - Issue: shell entered multiline prompt and produced syntax errors. - Impact: do not trust that SQLite inspection attempt. - Rule: avoid nested SSH heredoc/Python for this task; use simpler commands, generated nginx configs, or uploaded/local script files.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"d1f2dda1b36bc12803264780b71545d7d9b00c4bdb88ef2fa85405e9d27cdf6f\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L336-NPMPLUS-PUBLISH-SCRIPT-WRONG-ENV-SOURCE-525-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 336,\\n \\\"title\\\": \\\"NPMPLUS_PUBLISH_SCRIPT_WRONG_ENV_SOURCE_525_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 525 tried to create forum proxy hosts in NPMplus. - Issue: script sourced /opt/npmplus/.env, but INITIAL_ADMIN_EMAIL/PASSWORD were not present there; they are in the npmplus container env. - Impact: no forum proxy hosts were created by proof 525. - Rule: read NPMplus API login values from docker inspect env internally, never print them.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"58aa4eeee2841a42e270226a408177af9a099631e2a47857c825707b2ac523ff\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L342-NPMPLUS-API-LOGIN-MISSED-HTTPS81-AND-JSON-ESCAPE-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 342,\\n \\\"title\\\": \\\"NPMPLUS_API_LOGIN_MISSED_HTTPS81_AND_JSON_ESCAPE_20260701\\\",\\n \\\"summary\\\": \\\"- Context: NPMplus API login attempts in proofs 526/527 failed. - Issue: commands missed the likely HTTPS admin endpoint on port 81 and built JSON by string interpolation instead of jq escaping. - Impact: no proxy hosts were created by 526/527. - Rule: for NPMplus API use jq-generated JSON and test https://127.0.0.1:81/api/tokens with -k before publish.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"141ce6d2777003569c531b5308e56cffaea92dc113fdb0c9e9ebe45815805101\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L348-NPMPLUS-API-AUTH-UNAVAILABLE-MANUAL-ROUTES-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 348,\\n \\\"title\\\": \\\"NPMPLUS_API_AUTH_UNAVAILABLE_MANUAL_ROUTES_20260701\\\",\\n \\\"summary\\\": \\\"- Context: NPMplus API login attempts failed even though the forum VM and edge-to-forum connectivity are healthy. - Issue: API credentials from container initial env are not accepted by current NPMplus API. - Impact: do not use NPMplus API for this publish path. - Rule: publish forum routes through backed-up manual nginx proxy_host configs, then issue proper certificates after DNS points to edge.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"d5a4f4c6312277e9e2c1487b38c4bf554f4b9618fad9e2c5328fe8324e26a555\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L354-EDGE-CERTBOT-TOKEN-FILE-PERMISSION-535-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 354,\\n \\\"title\\\": \\\"EDGE_CERTBOT_TOKEN_FILE_PERMISSION_535_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 535 attempted DNS-01 certificate issue for five forum domains. - Issue: /tmp/forum_cf_token.env was root-owned mode 600, so debian user could not source it directly. - Impact: certbot ran without token and all certificate attempts failed; DNS cutover must not proceed from proof 535. - Rule: read Cloudflare token via sudo without printing it, then create temporary certbot credentials file and remove it after use.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"4566b4ef97181c31efcb32eefd3ed52633883fc929e98d4607a6c48671434582\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L360-EDGE-CERTBOT-FAILED-DNS-CUTOVER-ABORTED-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 360,\\n \\\"title\\\": \\\"EDGE_CERTBOT_FAILED_DNS_CUTOVER_ABORTED_20260701\\\",\\n \\\"summary\\\": \\\"- Context: edge manual routes are working locally, but certbot DNS-01 certificate issue failed in proof 538. - Issue: all five certbot attempts returned RC=1; proof 539 correctly aborted DNS cutover because edge cert precheck was not ready. - Impact: Cloudflare DNS records were not changed by proof 539; public DNS still points to old IP 87.236.18.45. - Rule: inspect certbot logs and Cloudflare token zone permissions before retrying certificate issuance or DNS cutover.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"7fc8883c0f715d038cc77e38a9088d6cb18290d2456232e9e0a13b1ba094d256\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L366-CLOUDFLARE-TOKEN-NO-ZONE-ACCESS-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 366,\\n \\\"title\\\": \\\"CLOUDFLARE_TOKEN_NO_ZONE_ACCESS_20260701\\\",\\n \\\"summary\\\": \\\"- Context: Cloudflare token discovered on edge verified as a token but could not access forum zones. - Evidence: proof 542 showed all five zones missing and DNS record create probes failed. - Impact: certbot DNS-01 failed and DNS cutover was correctly aborted; public DNS still points to old hosting. - Rule: install a new Cloudflare token with Zone Read and DNS Edit for the five forum zones before retrying certs/DNS.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"3ba4fdbb1dcc5d0beeb215153e4471c19e29f4080ff3d061e77154f44de9a898\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L372-FORUM-PUBLICATION-FINAL-SUCCESS-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 372,\\n \\\"title\\\": \\\"FORUM_PUBLICATION_FINAL_SUCCESS_20260701\\\",\\n \\\"summary\\\": \\\"- Context: five XenForo forums were rebuilt on VM160, routed through edge NPMplus, issued Let’s Encrypt certs using a corrected Cloudflare token, and cut over in DNS. - Result: final public proof 546 passed. - Important corrections: initial NPMplus API route failed; manual Nginx routes were used. Initial Cloudflare token had no zone access; corrected token passed zone/DNS probe.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"bec7980ce2230b8ecf69cfc694cc56bbd3dedc45da55ed2df567e8f41b228638\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L377-FINAL-SNAPSHOT-NAME-TOO-LONG-556-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 377,\\n \\\"title\\\": \\\"FINAL_SNAPSHOT_NAME_TOO_LONG_556_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 556 final health gate passed for all five public forums. - Issue: Proxmox snapshot failed because generated snapshot name exceeded 40 characters. - Evidence: qm snapshot returned snapname value may only be 40 characters long. - Impact: forum health was OK, but proof 556 snapshot step was not completed. - Fix: rerun snapshot with short name.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c3d65c37278c1e067af0f6dc1d7eb36fa6e75b03113162e66adf06c483bedc76\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L384-CLOUDFLARE-DNS-AUDIT-TXT-SUBSTRING-GAP-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 384,\\n \\\"title\\\": \\\"CLOUDFLARE_DNS_AUDIT_TXT_SUBSTRING_GAP_20260701\\\",\\n \\\"summary\\\": \\\"- Context: prior Cloudflare DNS audit checked web A/AAAA/CNAME state for root/www records. - Issue: it did not deeply inspect TXT/SPF contents for old provider IP substrings. - Evidence: user screenshot showed SPF TXT containing old 87.236.18.* value. - Impact: web routing is OK, but mail-related DNS may still contain stale provider data. - Rule: perform full DNS record audit including TXT/SPF/DKIM/DMARC/MX/CDN and substring search for old IPs before DNS cleanup.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"260c1f016ae9bf9efb5888bb54809e742681b512223781efac0a4cc8d8cdf2c7\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L391-CLOUDFLARE-DNS-CLEANUP-DRYRUN-QUOTE-563-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 391,\\n \\\"title\\\": \\\"CLOUDFLARE_DNS_CLEANUP_DRYRUN_QUOTE_563_20260701\\\",\\n \\\"summary\\\": \\\"- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt. - Impact: proof 563 is invalid and no DNS cleanup was performed by it. - Rule: use copied scripts for Cloudflare DNS mutation instead of fragile inline nested quotes.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"3837def1e15ccdc91af766587f44bd7e98885fb25b88ca3d62e65d13698f62d1\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L396-SPF-DUPLICATE-AFTER-565-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 396,\\n \\\"title\\\": \\\"SPF_DUPLICATE_AFTER_565_20260701\\\",\\n \\\"summary\\\": \\\"- Context: SPF cleanup command 565 attempted to replace stale SPF records. - Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained. - Impact: domains temporarily had duplicate SPF records, which is invalid for mail validation. - Fix: delete all SPF TXT records for the five forum zones, then create exactly one v=spf1 -all TXT per zone.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"4a4b1cba0ac945aef40797821d7a9a774728f30fb5e56c34599575e1b5b40ba8\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L402-MSMTP-SECRET-SOURCE-PARSE-ERROR-576-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 402,\\n \\\"title\\\": \\\"MSMTP_SECRET_SOURCE_PARSE_ERROR_576_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 576 installed msmtp but sendmail auth test failed. - Issue: [SENSITIVE_PATH] was sourced as shell, but SMTP password contained shell-special characters; source failed and auth used an invalid/truncated secret path. - Impact: msmtp package installed, but mail sending was not proven working. - Rule: never source SMTP secret files containing arbitrary passwords; store password base64 and use msmtp passwordeval helper accessible to www-data.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"6288f1fcabcdeda1884be858bea052ff9dbb35a65054f5cff6315a350dd7ef12\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L408-FORUM-MSMTP-MAIL-TRANSPORT-STILL-FAILING-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 408,\\n \\\"title\\\": \\\"FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701\\\",\\n \\\"summary\\\": \\\"- Context: attempted to fix msmtp config with passwordeval helper. - Issue: one or more sendmail/PHP mail tests still failed. - Impact: XenForo mail sending is not yet proven. - Next step: verify SMTP credential/password and provider policy for noreply@pvepro.ru.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"31d614a3bb69ec05a33a5be14388347a438e13f1e28a7ec58207273159e00eae\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L414-SMTP-PASSWORD-EXPOSED-ROTATE-REQUIRED-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 414,\\n \\\"title\\\": \\\"SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701\\\",\\n \\\"summary\\\": \\\"- Context: SMTP password was exposed in terminal output during failed msmtp setup. - Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line. - Impact: treat that SMTP password as compromised. - Required action: rotate the mailbox/app password in the mail provider panel before any further SMTP testing. - Rule: never store arbitrary passwords as shell-sourceable KEY=value files; use non-printing secret storage only.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"88ae0aeb13d948e6f2cb2d83dc2a9464e4de1805cdb4271cf78f3d315f7095c9\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L421-NETBIRD-MAILCOW-REPORTED-DOWN-AFTER-SMTP-INCIDENT-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 421,\\n \\\"title\\\": \\\"NETBIRD_MAILCOW_REPORTED_DOWN_AFTER_SMTP_INCIDENT_20260701\\\",\\n \\\"summary\\\": \\\"- Context: user reported NetBird and Mailcow became unavailable after forum mail-transport work. - Impact: treat as active incident until service reachability and container/VM state are proven. - Rule: do not change DNS, firewall, proxy, mail, NetBird, or Mailcow configs before read-only triage proof.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"b0eef26c0503e63d8a8c5a294ea1e124ab5e6ac8936d2ccaf23cd9b74fc5ebf3\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L426-FORUM-SMTP-FAILED-CONFIG-DISABLED-AFTER-MAILCOW-BAN-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 426,\\n \\\"title\\\": \\\"FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701\\\",\\n \\\"summary\\\": \\\"- Context: forum-prod SMTP/msmtp test used invalid or compromised credentials and triggered Mailcow netfilter warnings/ban for 95.84.154.183. - Impact: forum mail transport is not proven and must remain disabled until SMTP password is rotated and tested once carefully. - Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth. - Rule: do not test SMTP auth again until a fresh rotated mailbox/app password is available.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"70dc70d8c15e3f30611c45123bf2c3b27c86bf66d703df3e4232057d380801d6\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L432-FORUM-OLD-CODEVIPE-CLOUD-BACKUP-STALE-AFTER-FRESH5-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 432,\\n \\\"title\\\": \\\"FORUM_OLD_CODEVIPE_CLOUD_BACKUP_STALE_AFTER_FRESH5_20260701\\\",\\n \\\"summary\\\": \\\"- Context: pve02 codevipe-cloud-backup.timer existed before fresh five-forum rebuild. - Issue: old script points to /root/backups/codevipe and old remote path app/codevipe/forum-prod/local. - Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory" +} diff --git a/runtime/history/CONTEXT-SOURCES-FULL-REFRESH-363.txt b/runtime/history/CONTEXT-SOURCES-FULL-REFRESH-363.txt new file mode 100644 index 0000000..e8402e6 --- /dev/null +++ b/runtime/history/CONTEXT-SOURCES-FULL-REFRESH-363.txt @@ -0,0 +1,4094 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=CONTEXT-SOURCES-FULL-REFRESH-363 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-context +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=eb33834ec5df95efdbe55c4264ce02cbc69fcf90de62d457948a23d1e1efda04 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=9482ef04e0765ed01d6de068ebba4ef30fb0e133871b6cd45c4af4906d299122 +SANITIZED_OUTPUT_SHA256=3d9b0a662d873becba40d548b6854862d5ad518285238a73362573ff85988751 +OUTPUT_BEGIN +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/test +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/mktemp +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/rm +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/curl +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/jq +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/stat +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/sha256sum +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/awk +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/date +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/cat +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/printf +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/grep +DETAIL=STAGE=SOURCE_FETCH +DETAIL=IMMUTABLE_SOURCE=errors-index|SIZE=146628|SHA256=a13ca59d5fa1ad39b813bad3115d7b8cf8c5c68935fc813dbd792757ed421e24|STATUS=PASS +DETAIL=IMMUTABLE_SOURCE=workflow-contract|SIZE=1926|SHA256=1378a2c0d6dec6b09e3ea7dfced65fed1e1c38065baa6bba8bfd4c89e23e2c78|STATUS=PASS +DETAIL=IMMUTABLE_SOURCE=double-check-rule|SIZE=268|SHA256=7e5221154959bb0f435b3cfb955e6671d3c361ecc7a99625f04252cc74ebaa4b|STATUS=PASS +DETAIL=FULL_CONTEXT_BUNDLE_BEGIN +FULL_CONTEXT_BUNDLE_SCHEMA=1 +GENERATED_AT_UTC=2026-07-23T14:23:43Z +REFERENCE_FILE=/etc/pve/31_HOMELAB_REFERENCE.md +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +REFERENCE_SIZE=164602 +ERROR_REGISTER_FILE=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +ERROR_REGISTER_SIZE=79246 +AUTO_CONTEXT_ID=CONTEXT-AUTO-20260723T141652Z +AUTO_CONTEXT_COMMIT=fcf2ed2a6b11016ec6441a014f7832a86c51f793 +AUTO_CONTEXT_URL=https://git.gram1.ru/homelab-admin/homelab-public-context/raw/commit/fcf2ed2a6b11016ec6441a014f7832a86c51f793/runtime/history/CONTEXT-AUTO-20260723T141652Z.txt +AUTO_CONTEXT_SHA256=acd6a206ec5eb45ed50c2aae1cee02d4683dd8675e5c2d88190cefcd2446dada +AUTO_CONTEXT_SIZE=29049 +LATEST_JSON_AUTHORITATIVE=NO +DETAIL=SECTION_REFERENCE_REGISTER_BEGIN +HOMELAB REFERENCE +GENERATED=2026-06-29T21:45:29+03:00 +AUDIT_DIR=/root/cluster-audit-20260629T201245 + +CORE_CLUSTER_CONFIG + +keyboard: en-us +migration: secure,network=[PRIVATE_IP]/24 + +FINAL_CLASSIFICATION +MISSING_EXPECTED_PREFIX_COUNT=0 +STRICT_POSSIBLE_SECRET_COUNT=0 +STRICT_SECRET_SCAN_OK +HEALTH_WARN_ERROR_COUNT=14 +NOTE exclude_from_reference_or_delete_later=05_edge_compose_safe.tgz +NOTE forum-prod VMID160 is not in homelab-nightly-all; it has separate local/cloud backup chain +NOTE edge-vm disk scsi1 backup=0 risk must be documented +NOTE Corosync remains on [PRIVATE_IP]/12/13; migration network is [PRIVATE_IP]/24 secure +NOTE edge health WARN/ERROR items should be documented as known current states + +HEALTH_NOT_OK +HEALTH_NOT_OK item=adguard-rewrite-sync.txt line=STATUS=ERROR TS=20260629T152553Z TYPE=adguard-rewrite-sync REASON=URLError SECRET_PRINTED=REDACTED +HEALTH_NOT_OK item=backup-restore-coverage-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=backup-restore-dashboard.txt line=STATUS=WARN TS=2026-06-29T18:10:09Z ITEMS=80 NOT_OK=13 +HEALTH_NOT_OK item=container-image-lifecycle-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=cron-job-monitoring-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=direct-heartbeat-pilot-readiness.txt line=STATUS=WARN +HEALTH_NOT_OK item=healthchecks-heartbeat-coverage.txt line=STATUS=WARN +HEALTH_NOT_OK item=healthchecks-job-coverage.txt line=STATUS=WARN +HEALTH_NOT_OK item=healthchecks-job-monitoring-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=homepage-service-catalog-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=ingress-dns-route-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=job-monitoring-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=observability-health-surface-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=security-vulnerability-slo.txt line=STATUS=WARN + +EVIDENCE_FILES +00_cluster_overview.txt 3544 bytes +00_HOMELAB_ASSISTANT_ERROR_REGISTER.md 2566 bytes +01_pve01_audit.txt 28554 bytes +02_pve02_audit.txt 24786 bytes +03_pve03_audit.txt 16842 bytes +04_edge_vm_basic.txt 34506 bytes +05_edge_compose_redacted.txt 24228 bytes +05_edge_compose_safe.tgz.QUARANTINED.txt 354 bytes +05_edge_compose_tar_errors.txt 166 bytes +06_ACCESS_AND_ERROR_RULES.md 13742 bytes +06_edge_npmplus_routes_safe.txt 17350 bytes +07_edge_systemd_backup_audit.txt 20694 bytes +08_edge_scripts_redacted.txt 198622 bytes +09_forum_prod_basic.txt 14413 bytes +10_forum_codevipe_xenforo_audit.txt 3861 bytes +11_forum_backup_audit_redacted.txt 5952 bytes +12_pve02_codevipe_cloud_backup_audit.txt 6277 bytes +13_pve01_systemd_backup_audit.txt 60109 bytes +14_pve01_scripts_redacted.txt 246474 bytes +15_pve01_ct_110_audit.txt 5049 bytes +15_pve01_ct_112_audit.txt 4843 bytes +16_pve02_ct_111_audit.txt 4758 bytes +16_pve02_ct_113_audit.txt 4843 bytes +17_nextcloud_vm_audit.txt 13293 bytes +18_nextcloud_backup_restore_pve01_audit.txt 4269 bytes +19_pve02_host_automation_audit.txt 10642 bytes +19_pve03_host_automation_audit.txt 7632 bytes +20_AUDIT_INDEX_AND_SECRET_SCAN.txt 11935 bytes +21_proxmox_cluster_config_audit.txt 10855 bytes +22_internal_2_5g_network_inventory.txt 8036 bytes +23_cluster_internal_network_configured.txt 747 bytes +24_cluster_internal_network_speedtest.txt 54665 bytes +25_cluster_internal_network_migration_enabled.txt 1816 bytes +26_migration_network_pvesh_verify.txt 1210 bytes +27_migration_network_canonical_verify.txt 956 bytes +28_dns_configs_redacted.txt 15559 bytes +29_health_summary_all_nodes.txt 39051 bytes +30_FINAL_AUDIT_GAPS_WARNINGS_SECRET_CLASSIFICATION.txt 4366 bytes +31_HOMELAB_REFERENCE.md 1805 bytes +datacenter.cfg.before-canonical-migration-20260629T211046 63 bytes +datacenter.cfg.before-migration-network-20260629T210710 16 bytes + + + +РУССКОЕ_РЕЗЮМЕ_КЛАСТЕРА +- Кластер: homelab, 3 узла, quorum OK. +- LAN: pve01=[PRIVATE_IP], pve02=[PRIVATE_IP], pve03=[PRIVATE_IP]. +- Внутренняя сеть: pve01-int=[PRIVATE_IP], pve02-int=[PRIVATE_IP], pve03-int=[PRIVATE_IP]. +- Миграции Proxmox: migration: secure,network=[PRIVATE_IP]/24. +- Corosync остаётся на [PRIVATE_IP]/12/13; не переносить без отдельного rollback-плана. + +РИСКИ_И_ДЕЙСТВИЯ +- VM160 forum-prod не входит в ночной Proxmox backup. +- У VM130 edge-vm есть риск: дополнительный диск backup=0. +- Нужно ротировать ранее засвеченный Cloudflare token. +- Желательно ротировать XenForo SMTP пароль, который был выведен до редактирования. +- Проверить adguard-rewrite-sync: STATUS=ERROR, REASON=URLError. +- Проверить backup-restore-dashboard: STATUS=WARN, NOT_OK=13. + +ДОСТУПЫ_КРАТКО +- Proxmox: ssh root@pve01, ssh root@pve02, ssh root@pve03. +- Edge VM: ssh debian@[PRIVATE_IP], использовать sudo, root-login не использовать. +- Nextcloud VM: ssh debian@[PRIVATE_IP]. +- Forum-prod: сначала ssh root@pve02, затем ssh -i [SENSITIVE_PATH] root@[PRIVATE_IP]. + +НАГРУЗКИ_И_СЕРВИСЫ +- CT110 dns1 pve01 [PRIVATE_IP] AdGuard Home. +- CT111 dns2 pve02 [PRIVATE_IP] AdGuard Home. +- CT112 unbound1 pve01 [PRIVATE_IP] Unbound. +- CT113 unbound2 pve02 [PRIVATE_IP] Unbound. +- VM130 edge-vm pve03 [PRIVATE_IP] Docker ingress/app host. +- VM150 Nextcloud pve01 [PRIVATE_IP] Nextcloud AIO. +- VM160 forum-prod pve02 [PRIVATE_IP] CodeVipe/XenForo. + +BACKUP_КРАТКО +- Ночной Proxmox backup включает VMID 110,111,112,113,130,150. +- VM160 forum-prod исключён из ночного Proxmox backup и держится на app-level local/cloud backup. +- Edge VM имеет vzdump/offhost/restore evidence; риск backup=0 по дополнительному диску остаётся. +- Nextcloud VM150 имеет Proxmox backup и restore-proof evidence. +- CodeVipe forum-prod имеет локальный backup каждые 6 часов и cloud backup с pve02 каждые 6 часов. + +БЕЗОПАСНОСТЬ_КРАТКО +- Финальная строгая проверка audit-файлов: STRICT_POSSIBLE_SECRET_COUNT=0. +- Redacted DNS configs сохранены в 28_dns_configs_redacted.txt. +- Справочник 31 проверен: REFERENCE_ASSIGNMENT_SECRET_HITS=0. + +## FINAL_CLOSURE_20260630_CRITICAL_TAILS + +- VM160 backup: closed, nightly includes VM160, manual backup ZSTD_OK. +- VM130 backup: closed, scsi1 backup=1, manual backup ZSTD_OK, exact offhost ZSTD_OK, old local backup removed. +- AdGuard rewrite sync: closed, STATUS=OK, timer active/enabled. +- Cloudflare token: live NPMplus token rotated, API verify OK, old exposed token externally confirmed revoked. +- XenForo SMTP: password rotated, JSON valid, STARTTLS SMTP AUTH OK. +- Final audit: unredacted secret strict scan OK. +- Proof index: 85_FINAL_CLOSURE_PROOF.txt and 87_FINAL_CLOSURE_PROOF_WITH_EXTERNAL_CONFIRMATION.txt. + +## FINAL_DASHBOARD_RUNTIME_OK_20260630 + +- Runtime dashboard: STATUS=OK, ITEMS=69, NOT_OK=0. +- systemd failed units: 0 loaded units listed. +- SLO/coverage WARN files: moved out of runtime dashboard into slo-coverage-backlog-index.txt. +- Backlog index: STATUS=OK, BACKLOG_WARN_COUNT=12, RUNTIME_BLOCKER=NO. +- Final proof: 103_FINAL_ALL_GREEN_DASHBOARD_PROOF.txt. +## ROUTER_NETCRAZE_ULTRA_NC1812_20260630 + +Источник: self-test Netcraze Ultra NC-1812 от 2026-06-30. Секреты Wi-Fi, пароли и токены в справочник не вносятся. + +### Паспорт +- Model: Netcraze Ultra. +- Device description: Netcraze Ultra (NC-1812). +- Hostname: Netcraze-9202. +- Workgroup/domain: WORKGROUP. +- Timezone: Europe/Moscow. +- NTP: master. +- NDNS/caption: ndns-domain. +- Локальный домен/сертификат в логах: aleisaevn.netcraze.pro. +- Firmware/NDM из self-test: 5.01.C.0.0-1, built 2026-06-17. +- sharing-config version: 2.06.1. +- Components auto-update: disabled. +- Auto-update channel: draft. +- Auto-update schedule0: start 05:00, stop 06:00. +- EasyConfig: disabled. +- zram: enabled. +- IPv4 forwarding: enabled. +- IPv6 forwarding: enabled. +- conntrack max entries: 32768. +- TCP established timeout: 1200. +- TCP fin timeout: 30. +- TCP keepalive: 120. + +### WAN и резервный интернет +- Main WAN: GigabitEthernet1, renamed ISP, description Ростел. +- WAN security-level: public. +- WAN addressing: DHCP. +- WAN MTU: 1500. +- WAN global priority: 700. +- WAN ping-check profile: default. +- Ping-check hosts from log/profile: ya.ru, vk.com, dzen.ru over TLS/443. +- WAN DHCP observed in router log: 95.84.154.183/23, gateway 95.84.154.1. +- ISP DNS observed in router log: 77.37.251.33, 77.37.255.30. +- Backup/mobile WAN: CdcEthernet0, description SIM. +- CdcEthernet0 USB device-id: 12d1 14dc. +- CdcEthernet0 security-level: public. +- CdcEthernet0 addressing: DHCP. +- CdcEthernet0 global priority: 350. +- CdcEthernet0 observed IP: [PRIVATE_IP]/24 via [PRIVATE_IP]. + +### LAN / VLAN / bridge +- Home bridge: Bridge0 renamed Home. +- Home description: Основная. +- Home security-level: private. +- Home IP: [PRIVATE_IP]/24. +- Home includes: GigabitEthernet0/Vlan1, AccessPoint, AccessPoint_5G, XGigabitEthernet0. +- Proxmox bridge: Bridge1. +- Proxmox bridge IP: [PRIVATE_IP]/24. +- Proxmox security-level: protected. +- Proxmox includes: GigabitEthernet0/Vlan50, XGigabitEthernet0/Vlan50. +- Port 1: GigabitEthernet0/0, access VLAN 50. +- Port 2: GigabitEthernet0/1, access VLAN 1 + trunk VLAN 50. +- Port 3: GigabitEthernet0/2, access VLAN 1 + trunk VLAN 50. +- Port 4: GigabitEthernet0/3, access VLAN 1 + trunk VLAN 50. +- Port 5: XGigabitEthernet0/0, XGigabitEthernet0/Vlan50 enabled. + +### Wi-Fi +- SSID: N9202. +- 2.4 GHz: WifiMaster0, AccessPoint. +- 2.4 GHz compatibility: BGN+AX+BE. +- 2.4 GHz channel width: 40-below. +- 5 GHz: WifiMaster1, AccessPoint_5G. +- 5 GHz compatibility: AN+AC+AX+BE. +- 5 GHz channel width: 160. +- Auto channel rescan: 00:00 interval 1 hour. +- Encryption: WPA2 + WPA3. +- WMM: enabled. +- Beamforming: enabled. +- TWT: enabled. +- DL/UL MU-MIMO: enabled. +- DL/UL OFDMA: enabled. +- Spatial reuse: enabled. +- Band steering: disabled. +- Extra AP interfaces: present but down. +- MWS WLAN Home: bands 0 and 1, bind Home, SSID N9202, encryption wpa2+3. + +### DHCP +- Main DHCP pool: _WEBADMIN. +- DHCP range: [PRIVATE_IP]-[PRIVATE_IP]. +- Default router: [PRIVATE_IP]. +- DHCP DNS: [PRIVATE_IP], [PRIVATE_IP]. +- Lease: 25200 seconds. +- Bound interface: Home. +- Guest AP pool: _WEBADMIN_GUEST_AP enabled. + +### Static DHCP reservations +- [PRIVATE_IP] -> 00:e0:70:62:98:fb comp01. +- [PRIVATE_IP] -> 60:3d:61:d4:57:ea Yandex-Ambi-Lamp. +- [PRIVATE_IP] -> cc:4b:73:58:9b:c0 Yandex.Station. +- [PRIVATE_IP] -> 38:8a:06:5b:eb:40 Мой смарт. +- [PRIVATE_IP] -> a0:9d:c1:84:7b:98 Приставка комната. +- [PRIVATE_IP] -> ac:ba:c0:51:d3:ca Алиса наша комната. +- [PRIVATE_IP] -> 54:f1:5f:f3:cd:14 Приставка кухня. +- [PRIVATE_IP] -> ac:ba:c0:50:71:94 Алиса кухня. +- [PRIVATE_IP] -> 24:4b:fe:d8:8d:2c Коля смарт ROG-Phone3. +- [PRIVATE_IP] -> 00:2b:70:9a:a3:cf Коля смарт S21-FE. +- [PRIVATE_IP] -> 00:08:22:34:26:fc Коля SPIDER-10 первый. +- [PRIVATE_IP] -> 6c:c7:ec:93:be:dc Света смарт. +- [PRIVATE_IP] -> 00:1e:8f:b0:d5:af Принтер Canon ч/б. +- [PRIVATE_IP] -> d8:43:ae:68:a3:a1 Коля комп. +- [PRIVATE_IP] -> 80:1f:12:69:24:e6 Сигнализация. +- [PRIVATE_IP] -> 00:2b:70:9a:9e:99 Мой смарт S21-FE. +- [PRIVATE_IP] -> 74:eb:80:2e:aa:4e Света планшет Galaxy Tab S4. +- [PRIVATE_IP] -> 84:47:09:6a:cc:ca PVE01. +- [PRIVATE_IP] -> 6c:1f:f7:28:72:4f PVE02. +- [PRIVATE_IP] -> 08:26:ae:3b:73:61 PVE03. +- [PRIVATE_IP] -> bc:24:11:30:d4:0f AdGuard 1. +- [PRIVATE_IP] -> bc:24:11:5b:7e:d8 AdGuard 2. +- [PRIVATE_IP] -> bc:24:11:f4:c5:d8 WireGuard. +- [PRIVATE_IP] -> bc:24:11:e1:f3:3c NPMplus / edge-vm. +- [PRIVATE_IP] -> bc:24:11:e1:9a:25 Nextcloud. + +### NAT / port forwarding +- ISP tcp/80 -> bc:24:11:e1:f3:3c, NPMplus. +- ISP tcp/443 -> bc:24:11:e1:f3:3c, NPMplus. +- ISP tcpudp/3478 -> bc:24:11:e1:9a:25, Nextcloud Talk TURN. +- Home tcp/51820 -> bc:24:11:f4:c5:d8, WireGuard vpn1. + +### ACL / firewall +- isolate-private enabled. +- _WEBADMIN_Bridge0 permits Home-to-Proxmox access for DNS1/DNS2, AdGuard UI, NPMplus HTTP/HTTPS/UI, Nextcloud AIO/Talk TURN, Proxmox SSH/8006 and ICMP from admin PC. +- _WEBADMIN_GigabitEthernet1 permits admin PC [PRIVATE_IP] outbound TCP 80/443. +- Hotspot policy: Home permit, Bridge1 permit; known hosts explicitly permitted. + +### Router management +- HTTP port: 5080. +- HTTPS port: 5083. +- HTTP/HTTPS security-level: private. +- SSH port: 2222. +- SSH security-level: private. +- Telnet port configured: 2323, security-level private; router log shows Telnet disabled later on 2026-06-26. +- Lockout policy for HTTP/Telnet/SSH: 5 15 3. +- Cloud control2 security-level: private. +- Admin tags: cli, http, cifs, printers, opt. +- SFTP denied for admin in log; SSH CLI works. + +### Router services +- service dhcp enabled. +- service dns-proxy enabled. +- service igmp-proxy enabled. +- service http enabled. +- service cifs enabled. +- service ssh enabled. +- service ntp enabled. +- DNS proxy rebind protection: auto. +- mDNS reflector: disabled. +- UPnP LAN: Home. +- DLNA interface: Home. +- CIFS share: NTFS-edgtobxf, volume 6F98787822C01361:, automount, permissive. +- USB storage seen in log: Huawei modem/storage, sdb1 swap around 2G, sdb2 NTFS volume NTFS-edgtobxf. + +### Router automation warning +- Netcraze SSH CLI is not a normal POSIX shell. +- Logs contain failed commands: while, unset, follow. +- Automation must use router CLI syntax, not bash syntax. +## EXTERNAL_SERVICES_UPS_NETBIRD_MAIL_SCRIPTS_20260630 + +### UPS / NUT +- UPS hardware discovered on pve01 USB: Cyber Power System CP1500 AVR UPS, USB ID 0764:0501. +- pve01 role: NUT server + monitor. +- pve01 enabled NUT units: nut-driver-enumerator, nut-monitor, nut-server, nut-driver.target, nut.target. +- pve02 role: NUT monitor/client. +- pve03 role: NUT monitor/client. +- edge-vm: no UPS/NUT/APCUPSD integration discovered. +- UPS detailed proof: 107_UPS_DISCOVERY_AND_SETTINGS.txt and 109_UPS_NUT_DETAILED_STATUS_AND_CONFIGS.txt. +- Secrets from /etc/nut configs must remain redacted in audit output. + +### NetBird +- NetBird service is active on pve01, pve02, pve03 and edge-vm. +- NetBird version observed: daemon 0.73.2, CLI 0.73.2. +- pve01: FQDN pve01.netbird.selfhosted, IPv4 100.100.131.41/16. +- pve02: FQDN pve02.netbird.selfhosted, IPv4 100.100.7.2/16. +- pve03: FQDN pve03.netbird.selfhosted, IPv4 100.100.34.141/16. +- edge-vm: FQDN edge-vm.netbird.selfhosted, IPv4 100.100.60.182/16. +- Interface: wt0, type Kernel. +- WireGuard port: 51820. +- Management/Signal: Connected. +- Relays: 2/2 available. +- SSH Server through NetBird: Disabled. +- Observed peers count on listed hosts: 6/9 Connected. +- Inventory proof: 108_EXTERNAL_SERVICES_NETBIRD_MAIL_SCRIPTS_INVENTORY.txt. + +### Forum mail / SMTP +- forum-prod host: forum-prod.gram1.ru, VM IP [PRIVATE_IP]. +- Local MTA services were not discovered by the first inventory command; forum mail is treated as XenForo external SMTP, not local Postfix/Dovecot. +- XenForo SMTP rotation proof: 80_xenforo_smtp_rotation_closed.txt. +- SMTP auth proof: 79_forum_xenforo_smtp_auth_openssl_no_crlf_no_secret.txt. +- Safe current SMTP inventory: 110_XENFORO_MAIL_EXTERNAL_SMTP_SAFE_INVENTORY.txt. +- SMTP password must never be printed; store only host, port, SSL mode, auth flag, username length, password length and AUTH OK proof. + +### Scripts / automations +- Full script/unit inventory proof: 108_EXTERNAL_SERVICES_NETBIRD_MAIL_SCRIPTS_INVENTORY.txt. +- edge-vm owns most application health, dashboard, ingress, backup, NetBox, NPMplus, certificate, Trivy and vulnerability jobs. +- pve01 owns many backup/offhost/restore/health/security/NetBird VPS/rclone/sops/scrutiny jobs. +- pve02 owns CodeVipe cloud backup, Vaultwarden isolated restore drill, smartctl textfile and staging/rclone helpers. +- pve03 owns smartctl textfile, cluster internal IP, NetBird and staging/rclone helpers. +- All scripts must be treated as managed operational surface; every future edit needs proof file in the active cluster-audit directory. +## UPS_NUT_DETAILED_CONFIG_20260630 + +- USB discovery on pve01: Cyber Power System, Inc. CP1500 AVR UPS, USB ID 0764:0501. +- NUT logical UPS name: cyberpower. +- NUT-reported device model: BR1000ELCD. +- NUT manufacturer: CPS. +- Driver: usbhid-ups. +- NUT driver version: 2.8.1. +- CyberPower HID data version: 0.8. +- NUT USB vendorid/productid: 0764/0501. +- NUT server node: pve01. +- NUT server mode: MODE=netserver. +- NUT server listeners: 127.0.0.1:3493 and [PRIVATE_IP]:3493. +- NUT clients: pve02 and pve03 in MODE=netclient. +- pve01 monitor role: MONITOR cyberpower@localhost master. +- pve02 monitor role: MONITOR cyberpower@[PRIVATE_IP] slave. +- pve03 monitor role: MONITOR cyberpower@[PRIVATE_IP] slave. +- Current UPS status at inventory time: OL. +- Battery charge: 100%. +- Battery warning threshold: 20%. +- Battery low threshold: 10%. +- Runtime estimate: 2544 seconds. +- Runtime low threshold: 300 seconds. +- Battery type: PbAcid. +- Battery voltage: 12.9V nominal 12V. +- Input voltage: 221.0V nominal 230V. +- Output voltage: 221.0V. +- UPS load: 11%. +- Nominal real power: 600W. +- Beeper: disabled. +- Shutdown delay: 20 seconds. +- Start delay: 30 seconds. +- Shutdown command on monitored nodes: /sbin/shutdown -h +0. +- NUT timing: POLLFREQ=5, POLLFREQALERT=5, HOSTSYNC=15, DEADTIME=30, FINALDELAY=5. +- Powerdown flag: /etc/killpower. +- Secrets in /etc/nut/upsd.users and upsmon.conf are intentionally redacted in proofs. +- Proof file: 109_UPS_NUT_DETAILED_STATUS_AND_CONFIGS.txt. + +## XENFORO_EXTERNAL_SMTP_CURRENT_20260630 + +- Host: forum-prod.gram1.ru, VM IP [PRIVATE_IP]. +- Local MTA units: none discovered in inventory output. +- Mail mode: XenForo external SMTP, not local Postfix/Dovecot. +- SMTP_HOST=mail.pvepro.ru +- SMTP_PORT=587 +- SMTP_SSL=false +- SMTP_AUTH=login +- USERNAME_LEN=17 +- PASSWORD_LEN=30 +- SECRET_PRINTED=NO +- SMTP AUTH proof: 79_forum_xenforo_smtp_auth_openssl_no_crlf_no_secret.txt. +- Rotation closure proof: 80_xenforo_smtp_rotation_closed.txt. +## TIMERS_AND_SCRIPT_HASHES_REFERENCE_20260630 + +- Full timers and script SHA256 inventory: 114_TIMERS_AND_SCRIPT_HASHES_INVENTORY.txt. +- edge-vm timers cover runtime dashboard, Paperless guard, external canary, health metrics, restore drill index, AdGuard rewrite sync, NPMplus cert expiry, NetBox backup/sync, retention, cluster daily status, vulnerability and Trivy scans, ingress hardening and NPMplus admin bind. +- pve01 timers cover VPN/NetBird health, health metrics, smartctl, disk space, MkDocs refresh, VPS identity audit, storage capacity, quality gate, evidence catalog, backup freshness, docker health, Filebrowser backup/offhost/restore, NPMplus/Kuma backup, NetBird VPS backup/offhost, Authentik/Gitea/Vaultwarden backup, SOPS secret coverage, mail cloud upload/restore, Immich/Memos/Paperless backup/offhost/restore, auto backup, edge-vm vzdump, secret sanity. +- pve02 timers cover smartctl, CodeVipe cloud backup, dpkg-db backup and Vaultwarden isolated restore drill. +- pve03 timers cover smartctl and dpkg-db backup. +- Script hashes were captured for /usr/local/sbin and /usr/local/bin on edge-vm, pve01, pve02 and pve03. +- Future script edits must create a new proof file and refresh 114_TIMERS_AND_SCRIPT_HASHES_INVENTORY.txt. +## PROXMOX_CLUSTER_NODE_STORAGE_REFERENCE_20260630 + +### Cluster +- Cluster name: homelab. +- Nodes: 3. +- Quorum: OK / Quorate Yes. +- Expected votes: 3. +- Quorum threshold: 2. +- Transport: knet. +- Secure auth: on. +- Corosync membership: [PRIVATE_IP] pve01, [PRIVATE_IP] pve02, [PRIVATE_IP] pve03. +- Proxmox VE: pve-manager 9.2.3 on all three nodes. +- Debian: 13 / trixie on all three nodes. +- Kernel: 7.0.12-1-pve on all three nodes. +- Datacenter migration config: secure, network=[PRIVATE_IP]/24. +- Internal migration/cluster helper IPs: pve01 [PRIVATE_IP], pve02 [PRIVATE_IP], pve03 [PRIVATE_IP]. + +### Storage policy +- Storage local: dir /var/lib/vz, content iso/vztmpl/backup/import. +- Storage local-lvm: lvmthin pool data, content rootdir/images. +- pve01 local usage at inventory: 32.56%, local-lvm 17.50%. +- pve02 local usage at inventory: 12.86%, local-lvm 11.65%. +- pve03 local usage at inventory: 35.79%, local-lvm 64.84%. +- pve01 staging LV: /mnt/staging, 300G. +- pve02 staging LV: /mnt/staging, 150G. +- pve03 staging LV: /mnt/staging, 200G. +- pve03 local-lvm is the most constrained active VM storage because VM130 has 96G OS disk plus 150G media disk. + +### Nightly Proxmox backup job +- Job: homelab-nightly-all. +- Schedule: 03:30. +- Mode: snapshot. +- Compression: zstd. +- Storage: local. +- Enabled: yes. +- Mail notification: failure. +- Included VMIDs: 110,111,112,113,130,150,160. +- Retention: keep-daily=7, keep-weekly=4, keep-monthly=3. + +### Node pve01 +- FQDN: pve01.gram1.ru. +- LAN IP: [PRIVATE_IP]/24. +- Internal migration IP: [PRIVATE_IP]/24. +- NetBird IP: 100.100.131.41/16. +- Bridge: vmbr0 over nic0. +- CPU: Intel Core i9-12950HX, 24 logical CPUs. +- RAM: 31Gi. +- Disk: Lexar SSD NQ7A1 1TB. +- Main active workloads: CT110 dns1, CT112 unbound1, VM150 nextcloud. + +### Node pve02 +- FQDN: pve02.gram1.ru. +- LAN IP: [PRIVATE_IP]/24. +- Internal migration IP: [PRIVATE_IP]/24. +- NetBird IP: 100.100.7.2/16. +- Bridge: vmbr0 over nic2. +- CPU: Intel N100, 4 logical CPUs. +- RAM: 15Gi. +- Disk: SK800-1TB. +- Main active workloads: CT111 dns2, CT113 unbound2, VM160 forum-prod. + +### Node pve03 +- FQDN: pve03.gram1.ru. +- LAN IP: [PRIVATE_IP]/24. +- Internal migration IP: [PRIVATE_IP]/24. +- NetBird IP: 100.100.34.141/16. +- Bridge: vmbr0 over nic1. +- CPU: Intel Core i7-4900MQ, 8 logical CPUs. +- RAM: 31Gi. +- Disk: Samsung SSD 870 EVO 500GB. +- Main active workload: VM130 edge-vm. + +### Proof +- Full raw inventory: 120_PROXMOX_CLUSTER_NODE_STORAGE_NETWORK_INVENTORY.txt. +## PROXMOX_VM_CT_INVENTORY_REFERENCE_20260630 + +### CT110 dns1 +- Type: LXC. +- Node: pve01. +- Hostname/name: dns1. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:F9:3C:E1. +- Role: AdGuard Home DNS primary. +- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameserver: [PRIVATE_IP]. +- Onboot: yes. +- Startup order: 30. +- Unprivileged: yes. +- Backup job: included in homelab-nightly-all. + +### CT111 dns2 +- Type: LXC. +- Node: pve02. +- Hostname/name: dns2. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:CF:3F:66. +- Role: AdGuard Home DNS secondary. +- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameserver: [PRIVATE_IP]. +- Onboot: yes. +- Startup order: 30. +- Unprivileged: yes. +- Backup job: included in homelab-nightly-all. + +### CT112 unbound1 +- Type: LXC. +- Node: pve01. +- Hostname/name: unbound1. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:22:A6:0D. +- Role: Unbound recursive resolver primary. +- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameserver: [PRIVATE_IP]. +- Onboot: yes. +- Startup order: 20. +- Unprivileged: yes. +- Backup job: included in homelab-nightly-all. + +### CT113 unbound2 +- Type: LXC. +- Node: pve02. +- Hostname/name: unbound2. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:9E:AF:BE. +- Role: Unbound recursive resolver secondary. +- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameserver: [PRIVATE_IP]. +- Onboot: yes. +- Startup order: 20. +- Unprivileged: yes. +- Backup job: included in homelab-nightly-all. + +### VM130 edge-vm +- Type: QEMU VM. +- Node: pve03. +- Name: edge-vm. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:E1:F3:3C. +- User: debian. +- Role: edge application host / reverse proxy / monitoring / backup automation host. +- Resources: 4 cores, 12GiB RAM. +- Disks: scsi0 96G OS, scsi1 150G media/data. +- scsi1 backup flag: backup=1. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameservers: [PRIVATE_IP], [PRIVATE_IP]. +- QEMU guest agent: enabled. +- Onboot: yes. +- Startup order: 40. +- Backup job: included in homelab-nightly-all. +- Important note: VM130 has exact offhost backup proof after scsi1 backup=1. + +### VM150 nextcloud +- Type: QEMU VM. +- Node: pve01. +- Name: nextcloud. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:E1:9A:25. +- User: debian. +- Role: Nextcloud AIO. +- Resources: 4 cores, 8GiB RAM. +- Disk: scsi0 64G. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameservers: [PRIVATE_IP], [PRIVATE_IP]. +- QEMU guest agent: enabled. +- Onboot: yes. +- Startup order: 50. +- Backup job: included in homelab-nightly-all. + +### VM160 forum-prod +- Type: QEMU VM. +- Node: pve02. +- Name: forum-prod. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:B6:45:ED. +- User: ops. +- Role: CodeVipe / XenForo production forum. +- Resources: 2 cores, 4GiB RAM. +- Disk: scsi0 80G. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameserver in VM config: 1.1.1.1. +- QEMU guest agent: enabled. +- Onboot: yes. +- Startup order: 30. +- Backup job: included in homelab-nightly-all. +- Important note: VM160 manual backup proof exists and VM160 is included in nightly job. + +### Proof +- Redacted config inventory: 121_PROXMOX_VM_CT_CONFIG_INVENTORY_REDACTED.txt. +## DNS_INGRESS_CERTIFICATES_REFERENCE_20260630 + +### DNS chain +- Home DHCP gives clients DNS servers: [PRIVATE_IP] and [PRIVATE_IP]. +- dns1: CT110, AdGuard Home, IP [PRIVATE_IP]. +- dns2: CT111, AdGuard Home, IP [PRIVATE_IP]. +- unbound1: CT112, IP [PRIVATE_IP], listens on [PRIVATE_IP]:5335 and 127.0.0.1:5335. +- unbound2: CT113, IP [PRIVATE_IP], listens on [PRIVATE_IP]:5335 and 127.0.0.1:5335. +- dns1 upstream: [PRIVATE_IP]:5335. +- dns2 upstream: [PRIVATE_IP]:5335. +- AdGuard bootstrap DNS: 1.1.1.1 and 9.9.9.9. +- AdGuard filtering_enabled=true, rewrites_enabled=true, protection_enabled=true. +- AdGuard ratelimit=20. +- Unbound access model: localhost allowed, matching AdGuard IP allowed, rest of [PRIVATE_IP]/24 refused. +- Unbound do-ip6: no. +- Unbound private-address includes [PRIVATE_IP]/16, [PRIVATE_IP]/12 and [PRIVATE_IP]/8. + +### AdGuard rewrites +- turn.gram1.ru -> [PRIVATE_IP]. +- git.gram1.ru -> [PRIVATE_IP]. +- dozzle.gram1.ru -> [PRIVATE_IP]. +- paper.gram1.ru -> [PRIVATE_IP]. +- memos.gram1.ru -> [PRIVATE_IP]. +- photos.gram1.ru -> [PRIVATE_IP]. +- auth.gram1.ru -> [PRIVATE_IP]. +- backup.gram1.ru -> [PRIVATE_IP]. +- Rewrite sync health proof: adguard-rewrite-sync STATUS=OK, ORIGIN_COUNT=8, REPLICA_COUNT=8, CHANGED=0. +- Proof files: 127_ADGUARD_UNBOUND_DNS_SAFE_INVENTORY.txt and 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt. + +### Router ingress +- Public WAN router forwards TCP/80 and TCP/443 to NPMplus on edge-vm, [PRIVATE_IP]. +- Router forwards TCP/UDP 3478 to Nextcloud Talk TURN on [PRIVATE_IP]. +- Router forwards Home TCP/51820 to WireGuard host [PRIVATE_IP]. +- NPMplus admin listener is bound to localhost on edge-vm, 127.0.0.1:81; public disabled legacy host npm.gram1.ru exists but enabled=0. + +### NPMplus runtime +- Host: edge-vm, [PRIVATE_IP]. +- Container: npmplus, image zoeyvid/npmplus:2026-06-17-b1, healthy at inventory time. +- Socket proxy: lscr.io/linuxserver/socket-proxy:latest, 127.0.0.1:2375. +- Database: /opt/npmplus/npmplus/database.sqlite. +- DB integrity: ok. +- Public listen ports on edge-vm: 0.0.0.0:80 and 0.0.0.0:443 by nginx/NPMplus. +- NPMplus admin: 127.0.0.1:81. +- Secret rule: Cloudflare DNS API token exists only inside NPMplus certificate metadata and must never be printed. + +### Public NPMplus proxy hosts +- nc.gram1.ru -> http://[PRIVATE_IP]:11000, cert=17, ssl_forced=1, enabled=1. +- uptime.gram1.ru -> http://127.0.0.1:3001, cert=35, ssl_forced=1, enabled=1. +- gotify.gram1.ru -> http://127.0.0.1:8082, cert=35, ssl_forced=1, enabled=1. +- vault.gram1.ru -> http://127.0.0.1:8083, cert=21, ssl_forced=1, enabled=1. +- dockge.gram1.ru -> http://127.0.0.1:5001, cert=26, ssl_forced=1, enabled=1. +- home.gram1.ru -> http://127.0.0.1:3000, cert=27, ssl_forced=1, enabled=1. +- git.gram1.ru -> http://127.0.0.1:3002, cert=29, ssl_forced=1, enabled=1. +- dozzle.gram1.ru -> http://127.0.0.1:9999, cert=30, ssl_forced=1, enabled=1. +- paper.gram1.ru -> http://127.0.0.1:8010, cert=30, ssl_forced=1, enabled=1. +- memos.gram1.ru -> http://127.0.0.1:5230, cert=30, ssl_forced=1, enabled=1. +- photos.gram1.ru -> http://127.0.0.1:2283, cert=30, ssl_forced=1, enabled=1. +- auth.gram1.ru -> http://127.0.0.1:9000, cert=30, ssl_forced=1, enabled=1. +- backup.gram1.ru -> http://[PRIVATE_IP]:9101, cert=31, ssl_forced=1, enabled=1. + +### VPN NPMplus proxy hosts +- kuma.vpn.gram1.ru -> http://127.0.0.1:3001, cert=32. +- homepage.vpn.gram1.ru -> http://127.0.0.1:3000, cert=32. +- gitea.vpn.gram1.ru -> http://127.0.0.1:3002, cert=32. +- filebrowser.vpn.gram1.ru -> http://127.0.0.1:8085, cert=32. +- auth.vpn.gram1.ru -> http://127.0.0.1:9000, cert=32. +- actual-budget.vpn.gram1.ru -> http://127.0.0.1:5006, cert=32. +- grafana.vpn.gram1.ru -> http://127.0.0.1:3003, cert=32. +- homebox.vpn.gram1.ru -> http://127.0.0.1:7745, cert=32. +- mealie.vpn.gram1.ru -> http://127.0.0.1:9001, cert=32. +- n8n.vpn.gram1.ru -> http://127.0.0.1:5678, cert=32. +- netbox.vpn.gram1.ru -> http://127.0.0.1:8000, cert=32. +- node-red.vpn.gram1.ru -> http://127.0.0.1:1880, cert=32. +- beszel.vpn.gram1.ru -> http://127.0.0.1:8090, cert=32. +- it-tools.vpn.gram1.ru -> http://127.0.0.1:8084, cert=32. +- karakeep.vpn.gram1.ru -> http://127.0.0.1:3030, cert=32. +- linkding.vpn.gram1.ru -> http://127.0.0.1:9091, cert=32. +- minio.vpn.gram1.ru -> http://127.0.0.1:9003, cert=32. +- ntfy.vpn.gram1.ru -> http://127.0.0.1:8055, cert=32. +- searxng.vpn.gram1.ru -> http://127.0.0.1:8888, cert=32. +- syncthing.vpn.gram1.ru -> http://127.0.0.1:8384, cert=32. +- alertmanager.vpn.gram1.ru -> http://127.0.0.1:9093, cert=32. +- prometheus.vpn.gram1.ru -> http://127.0.0.1:9090, cert=32. +- homeassistant.vpn.gram1.ru -> http://127.0.0.1:8123, cert=32. +- npmplus.vpn.gram1.ru -> https://[PRIVATE_IP]:81, cert=32. +- proxmox.vpn.gram1.ru -> https://[PRIVATE_IP]:8006, cert=32. +- dns1.vpn.gram1.ru -> http://[PRIVATE_IP]:8080, cert=32. +- dns2.vpn.gram1.ru -> http://[PRIVATE_IP]:8080, cert=32. +- checks.vpn.gram1.ru -> http://127.0.0.1:8015, cert=32. +- vikunja.vpn.gram1.ru -> http://127.0.0.1:8016, cert=32. +- bookstack.vpn.gram1.ru -> http://127.0.0.1:8017, cert=32. +- pdf.vpn.gram1.ru -> http://127.0.0.1:8018, cert=32. +- jellyfin.vpn.gram1.ru -> http://127.0.0.1:8019, cert=32. +- audiobooks.vpn.gram1.ru -> http://127.0.0.1:8020, cert=32. +- books.vpn.gram1.ru -> http://127.0.0.1:8021, cert=32. + +### NPMplus certificates +- cert=17: nc.gram1.ru, expires 2026-09-12 21:58:35. +- cert=21: vault.gram1.ru, expires 2026-09-13 13:43:23. +- cert=26: dockge.gram1.ru, expires 2026-09-13 14:51:59. +- cert=27: home.gram1.ru, expires 2026-09-13 16:43:44. +- cert=29: git.gram1.ru, expires 2026-09-13 17:36:55. +- cert=30: dozzle.gram1.ru, paper.gram1.ru, memos.gram1.ru, photos.gram1.ru, auth.gram1.ru, expires 2026-09-13 23:12:00. +- cert=31: backup.gram1.ru, expires 2026-09-14 05:04:53. +- cert=32: *.vpn.gram1.ru and vpn.gram1.ru, expires 2026-09-15 23:18:29. +- cert=35: uptime.gram1.ru and gotify.gram1.ru, expires 2026-09-19 21:21:59. +- Deleted/old cert rows observed: cert=1 nc.gram1.ru, cert=18 vault.gram1.ru. + +### Proof +- NPMplus schema/listen inventory: 126_DNS_INGRESS_CERT_NPMPLUS_SAFE_INVENTORY.txt. +- NPMplus exact proxy/cert rows: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt. +- AdGuard/Unbound inventory: 127_ADGUARD_UNBOUND_DNS_SAFE_INVENTORY.txt. +- AdGuard upstream/rewrites and Unbound rules: 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt. +## EDGE_DOCKER_STACKS_REFERENCE_20260630 + +### Runtime +- Host: edge-vm, IP [PRIVATE_IP]. +- Docker Server version: 29.6.0. +- Docker Compose version: v5.1.4. +- Primary stack root: /opt/stacks. +- Additional compose roots: /opt/npmplus-compose, /opt/gotify-compose, /opt/uptime-kuma-compose, /opt/vaultwarden-compose, /opt/dockge-compose. +- Public ingress terminates through NPMplus on ports 80/443. +- Most app containers expose only 127.0.0.1 ports and are published through NPMplus. +- NPMplus uses host networking. +- socket-proxy exposes Docker API read-oriented surface on 127.0.0.1:2375 and socket-proxy_net. +- Secret values are not stored in the reference. Only .env/secret file paths are inventoried. + +### Compose projects observed +- actual-budget. +- audiobookshelf. +- authentik. +- beszel. +- blackbox-exporter. +- bookstack. +- cadvisor. +- calibre-web. +- crowdsec. +- diun. +- dockge-compose. +- dozzle. +- filebrowser. +- gitea. +- gotify-compose. +- healthchecks. +- homeassistant. +- homebox. +- homepage. +- immich. +- it-tools. +- jellyfin. +- karakeep. +- kopia. +- linkding. +- loki-alloy. +- mealie. +- memos. +- minio. +- n8n. +- netbox. +- node-red. +- npmplus. +- ntfy. +- observability-lite. +- paperless. +- searxng. +- socket-proxy. +- stirling-pdf. +- syncthing. +- uptime-kuma-compose. +- vaultwarden-compose. +- vikunja. + +### Critical app groups +- Ingress/security: npmplus, socket-proxy, crowdsec. +- Identity/secrets: authentik, vaultwarden. +- Monitoring/alerting: uptime-kuma, gotify, healthchecks, observability-lite, loki-alloy, blackbox-exporter, cadvisor, node-exporter, dozzle, diun. +- Data apps: immich, paperless, memos, gitea, netbox, actual-budget, homebox, mealie, n8n, node-red, vikunja, bookstack. +- Files/media: nextcloud external VM, immich, filebrowser, syncthing, jellyfin, audiobookshelf, calibre-web, stirling-pdf, minio. +- Admin/dashboard: homepage, dockge, netbox, dozzle, NPMplus VPN admin. + +### Notable local ports +- Homepage: 127.0.0.1:3000. +- Uptime Kuma: 127.0.0.1:3001. +- Gitea: 127.0.0.1:3002. +- Grafana: 127.0.0.1:3003. +- Actual Budget: 127.0.0.1:5006. +- n8n: 127.0.0.1:5678. +- Paperless: 127.0.0.1:8010. +- Healthchecks: 127.0.0.1:8015. +- Vikunja: 127.0.0.1:8016. +- BookStack: 127.0.0.1:8017. +- Stirling PDF: 127.0.0.1:8018. +- Jellyfin: 127.0.0.1:8019. +- Audiobookshelf: 127.0.0.1:8020. +- Calibre-Web: 127.0.0.1:8021. +- ntfy: 127.0.0.1:8055. +- Gotify: 127.0.0.1:8082. +- Vaultwarden: 127.0.0.1:8083. +- IT-Tools: 127.0.0.1:8084. +- Filebrowser: 127.0.0.1:8085. +- Beszel: 127.0.0.1:8090. +- Prometheus: 127.0.0.1:9090. +- Linkding: 127.0.0.1:9091. +- Alertmanager: 127.0.0.1:9093. +- Node exporter: 127.0.0.1:9100. +- Blackbox exporter: 127.0.0.1:9115. +- Syncthing UI: 127.0.0.1:8384. +- Loki: 127.0.0.1:3100. +- Alloy UI: 127.0.0.1:12345. +- Home Assistant: host network, 0.0.0.0:8123. +- NPMplus admin: 127.0.0.1:81. +- NPMplus public ingress: 0.0.0.0:80 and 0.0.0.0:443. + +### Secret/env inventory policy +- Full values from .env, secrets.yaml, secretkey.txt and token files must never be copied into reference. +- Reference may list paths only. +- Full path inventory is in 133_EDGE_DOCKER_STACKS_SAFE_INVENTORY.txt. +- Normalized runtime map is in 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt. + +### Proof +- Raw Docker/stacks inventory: 133_EDGE_DOCKER_STACKS_SAFE_INVENTORY.txt. +- Normalized container map: 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt. +## BACKUP_RESTORE_OFFHOST_CATALOG_REFERENCE_20260630 + +### Global backup model +- Primary Proxmox backup job: homelab-nightly-all. +- Schedule: daily 03:30. +- Mode: snapshot. +- Compression: zstd. +- Storage: local. +- Retention: keep-daily=7, keep-weekly=4, keep-monthly=3. +- Included VMIDs: 110,111,112,113,130,150,160. +- Backup health dashboard: STATUS=OK, ITEMS=69, NOT_OK=0 at 2026-06-30T10:10:06Z. +- Restore drill index: STATUS=OK, RESTORE_PROOFS=16/16. +- SLO coverage is tracked separately from runtime dashboard. It had WARN backlog in backup-restore-coverage-slo with APP_TOTAL=43, GREEN=3, YELLOW=21, RED=19, MISSING_BACKUP_SIGNAL=31, MISSING_RESTORE_SIGNAL=28, MISSING_TIMER=31. +- Runtime OK means operational backup/restore checks are green; it does not mean every P1/P2 application has full backup+restore automation coverage. + +### Proxmox vzdump catalog +- CT110 dns1: included in homelab-nightly-all, local backup on pve01. +- CT111 dns2: included in homelab-nightly-all, local backup on pve02. +- CT112 unbound1: included in homelab-nightly-all, local backup on pve01. +- CT113 unbound2: included in homelab-nightly-all, local backup on pve02. +- VM130 edge-vm: included in homelab-nightly-all, local backup on pve03. +- VM150 nextcloud: included in homelab-nightly-all, local backup on pve01. +- VM160 forum-prod: included in homelab-nightly-all, local backup on pve02. +- VM130 also has dedicated edge-vm-vzdump backup/offhost/restore health proofs. +- VM160 has manual backup proof and nightly job inclusion proof. +- VM130 scsi1 backup flag is enabled after correction: scsi1 backup=1. + +### Edge VM / VM130 full-image protection +- edge-vm-vzdump-backup: STATUS=OK, archive size about 28.2G, SHA256 recorded. +- edge-vm-vzdump-offhost: STATUS=OK, destination pve02 /mnt/staging/offhost/edge-vm-vzdump-from-pve03. +- edge-vm-vzdump-restore: STATUS=OK, zstd and vma verification OK. +- mail-cloud-edge-vm: STATUS=OK, recurring chunked upload, 53 parts, download verification enabled. +- Retention for edge-vm cloud upload: RETENTION_KEEP=4. +- Edge restore proof offhost: STATUS=OK, ITEMS=14, OK=14. + +### Mail/cloud critical backups +- mail-cloud-critical: STATUS=OK, recurring P0 critical baseline, remote OK 2/2. +- Latest observed critical bundle: p0-critical-baseline-20260630T034644Z.tar.gz. +- Critical bundle size at inventory: 1087208837 bytes. +- Critical retention: RETENTION_KEEP=14. +- mail-cloud-restore-drill: STATUS=OK, CRITICAL_OK=2, EDGE_OK=1. +- vaultwarden isolated restore drill on pve02: STATUS=OK, uses cloud critical baseline plus pve02 offhost component and vaultwarden archive. + +### NPMplus / Kuma / ingress config backups +- npmplus-kuma-config-backup: STATUS=OK. +- Archive: /mnt/staging/npmplus-kuma-config-backups/snapshots/npmplus-kuma-config-*.tar.gz. +- NPMplus DB integrity: OK. +- Kuma DB integrity: OK. +- NPM proxy count in health proof: 47. +- Required VPN routes in health proof: 18. +- Wildcard VPN cert check: NPM_CERT32_WILDCARD=1. +- npmplus-kuma-config-offhost: STATUS=OK to pve02. +- npmplus-kuma-config-restore: STATUS=OK with DB integrity checks. +- npmplus restore proof also exists from app backup quality checks. + +### DNS / AdGuard / Unbound protection +- AdGuard rewrite sync: STATUS=OK, ORIGIN_COUNT=8, REPLICA_COUNT=8, CHANGED=0. +- dns1/dns2/unbound1/unbound2 are protected by Proxmox vzdump. +- AdGuard rewrite sync plan stored under /var/lib/homelab-private/adguard-rewrite-sync. +- Router self-test config is stored as uploaded/generated reference evidence, not yet automated as recurring router backup. + +### Auth / secrets / identity apps +- Vaultwarden backup: STATUS=OK, archive under /mnt/staging/vaultwarden-backups/snapshots. +- Vaultwarden offhost: STATUS=OK to pve02. +- Vaultwarden restore: STATUS=OK, DB integrity OK, tables counted. +- Vaultwarden isolated restore drill: STATUS=OK on pve02. +- Authentik backup: STATUS=OK, archive under /mnt/staging/authentik-backups/snapshots. +- Authentik offhost: STATUS=OK to pve02. +- Authentik restore: STATUS=OK, tables/users/applications/proxy providers checked. +- Secret values must remain out of all reference files; only archive paths, sizes and SHA256 are recorded. + +### Git / documentation / inventory apps +- Gitea backup: STATUS=OK, archive under /mnt/staging/gitea-backups/snapshots. +- Gitea offhost: STATUS=OK to pve02. +- Gitea restore: STATUS=OK, DB integrity OK, tables counted. +- NetBox backup: STATUS=OK, PostgreSQL dump under /var/lib/homelab-private/pre-update-backups/netbox-*. +- NetBox offhost: STATUS=OK to pve02. +- NetBox restore dry-run: STATUS=OK, restore container checked. +- Latest NetBox health proof observed: devices=7, sites=1, prefixes=3, IP addresses=12, virtual machines=49. + +### Document / notes / personal data apps +- Paperless backup: STATUS=OK, latest snapshot under /mnt/staging/paperless-backups/snapshots. +- Paperless offhost: STATUS=OK to pve02. +- Paperless restore dry-run: STATUS=OK, pg_restore list and restore tables checked. +- Memos backup: STATUS=OK, latest snapshot under /mnt/staging/memos-backups/snapshots. +- Memos offhost: STATUS=OK to pve02. +- Memos restore proof exists from app restore quality checks. +- Actual Budget backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof. +- Linkding backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof. +- Karakeep backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof. +- Mealie backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof. +- n8n backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof. + +### Files / media / sync apps +- Immich media backup: STATUS=OK, latest snapshot under /mnt/staging/immich-media-backups/snapshots. +- Immich media offhost: STATUS=OK to pve02. +- Immich media restore: STATUS=OK, local/offhost manifest match. +- Immich full consistency backup: STATUS=OK, includes PostgreSQL dump and media archive, offhost path present, restore dry-run OK. +- Nextcloud restore proof: STATUS=OK, OCC OK, DB_DUMP OK, DATA_MANIFEST OK, restore to production NO. +- Nextcloud restore proof is copied offhost. +- Filebrowser backup: STATUS=OK. +- Filebrowser offhost: STATUS=OK. +- Filebrowser restore validation: STATUS=OK. +- Jellyfin restore: STATUS=OK from app backup quality checks. +- Audiobookshelf restore: STATUS=OK from app backup quality checks. +- Calibre-Web restore: STATUS=OK from app backup quality checks. +- Syncthing is inventoried in Docker/stacks; dedicated backup signal not confirmed in this catalog. + +### Home/admin/utility apps +- Home Assistant backup: STATUS=OK, archive includes configuration.yaml, storage dir and DB file, offhost SHA256/listing OK, restore not attempted in observed proof. +- Healthchecks restore: STATUS=OK, SQLite integrity OK and temp restore verify OK. +- BookStack restore: STATUS=OK, DB dump OK. +- Vikunja restore: STATUS=OK, SQLite integrity OK and temp restore verify OK. +- Kopia restore: STATUS=OK, synthetic snapshot and restore compare OK. +- Observability config backup: STATUS=OK, includes prometheus.yml, alertmanager.yml and health rules, offhost SHA256/listing OK. +- Ntfy is covered as part of observability/alerting stack but dedicated service backup signal was not confirmed in this catalog. +- MinIO is included in app-stacks-batch backup from 20260617; current dedicated timer was not confirmed. +- Homebox, Node-RED, MinIO, Mealie, Linkding, n8n and Actual Budget were part of app-stacks batch backup proof from 20260617. + +### External service backups +- NetBird VPS backup: STATUS=OK, snapshot under /mnt/staging/netbird-vps-backups/snapshots. +- NetBird VPS offhost: STATUS=OK to pve02. +- NetBird VPS restore validation: STATUS=OK, archive SHA256 recorded. +- CodeVipe/forum-prod: VM160 Proxmox backup is present; pve02 also has codevipe-cloud-backup.timer. +- XenForo SMTP rotation has DB backup proof from earlier rotation work; production forum application-level backup catalog should be expanded separately with CodeVipe-specific DB/files/cron/public cutover details. + +### Retention and cleanup +- Retention dry-run: STATUS=OK, production changed NO, destructive actions NO. +- Disk retention policy: STATUS=OK, root used pct observed 70 on edge-vm, removed dirs 0, Docker volume prune NO. +- App backup retention dry-run timer exists on edge-vm. +- homelab-backup-freshness timer exists on pve01. +- rollback-image-retention health file exists but had permission issue during one raw read; it must be read with sudo in future collection commands. + +### Known coverage gaps / backlog +- backup-restore-coverage-slo remains a coverage backlog, not a runtime blocker. +- Actual Budget latest proof says RESTORE_ATTEMPTED=NO. +- Home Assistant latest proof says RESTORE_ATTEMPTED=NO. +- Linkding latest proof says RESTORE_ATTEMPTED=NO. +- Karakeep latest proof says RESTORE_ATTEMPTED=NO. +- Mealie latest proof says RESTORE_ATTEMPTED=NO. +- n8n latest proof says RESTORE_ATTEMPTED=NO. +- Observability config latest proof says RESTORE_ATTEMPTED=NO. +- Router config backup is currently reference/self-test based, not confirmed as recurring automated backup. +- Syncthing, ntfy, MinIO and several P2 services need either dedicated backup/restore proof or explicit classification as disposable/rebuildable. + +### Proof +- Raw backup/offhost/restore inventory: 138_BACKUP_RESTORE_OFFHOST_CATALOG_RAW_INVENTORY.txt. +- Backup timer summary draft: 139_BACKUP_CATALOG_SERVICE_SUMMARY_DRAFT.txt. +- Normalized backup health index: 140_BACKUP_HEALTH_FILES_NORMALIZED_INDEX.txt. +## MONITORING_ALERTING_HEALTH_REFERENCE_20260630 + +### Monitoring stack +- Primary monitoring host: edge-vm, [PRIVATE_IP]. +- Prometheus container: prometheus, local port 127.0.0.1:9090. +- Grafana container: grafana, local port 127.0.0.1:3003, VPN route grafana.vpn.gram1.ru. +- Alertmanager container: alertmanager, local port 127.0.0.1:9093. +- Loki container: loki, local port 127.0.0.1:3100. +- Alloy container: alloy, local port 127.0.0.1:12345. +- Uptime Kuma container: uptime-kuma, local port 127.0.0.1:3001, public route uptime.gram1.ru and VPN route kuma.vpn.gram1.ru. +- Gotify container: gotify, local port 127.0.0.1:8082, public route gotify.gram1.ru. +- ntfy container: ntfy, local port 127.0.0.1:8055, VPN route ntfy.vpn.gram1.ru. +- Healthchecks container: healthchecks, local port 127.0.0.1:8015, VPN route checks.vpn.gram1.ru. +- Blackbox exporter: blackbox-exporter, local port 127.0.0.1:9115. +- cAdvisor: cadvisor, local port 127.0.0.1:8081. +- Node exporter on edge-vm: node-exporter, local port 127.0.0.1:9100. +- Beszel: beszel, local port 127.0.0.1:8090. +- Dozzle: dozzle, local port 127.0.0.1:9999. + +### PVE monitoring endpoints +- pve01: prometheus-node-exporter active on :9100, homelab-health-http active on :9101, Proxmox pveproxy on :8006. +- pve02: prometheus-node-exporter active on :9100, homelab-health-http active on :9101, Proxmox pveproxy on :8006. +- pve03: prometheus-node-exporter active on :9100, Proxmox pveproxy on :8006. +- pve03 has node-exporter/SMART timers but no homelab-health-http :9101 endpoint in observed output. +- PVE smartctl textfile timers exist on pve01/pve02/pve03. +- pve01 also runs private VPN host health, NetBird peer health, disk space health, MkDocs refresh, evidence catalog and quality gate timers. + +### Prometheus config +- Prometheus scrape interval: 30s. +- Prometheus evaluation interval: 30s. +- Rule files path: /etc/prometheus/rules/*.yml. +- Alertmanager target: alertmanager:9093. +- Blackbox HTTP job targets: + - https://nc.gram1.ru + - https://git.gram1.ru + - https://auth.gram1.ru + - https://paper.gram1.ru + - https://backup.gram1.ru +- Edge node exporter scrape target: node-exporter:9100. +- PVE node exporter targets: [PRIVATE_IP]:9100, [PRIVATE_IP]:9100, [PRIVATE_IP]:9100. +- Proxmox exporter job targets pve01/pve02/pve03 through exporter endpoint [PRIVATE_IP]:9221. +- cAdvisor scrape target: cadvisor:8080. + +### Alertmanager / notification routing +- Alertmanager route receiver: ntfy. +- Alertmanager webhook target: http://ntfy/homelab-alerts. +- Alert group_by: alertname, instance, severity. +- group_wait: 10s. +- group_interval: 5m. +- repeat_interval: 4h. +- Alert routing proof: alert-routing.txt STATUS=OK, RULES_ADDED=6, PROMETHEUS_RELOAD=OK, TEST_NOTIFICATION_SENT=YES, NTFY_MESSAGES_PUBLISHED_DELTA=1. +- alert-routing-health.txt is the standardized status alias and is STATUS=OK. + +### Core Prometheus alert rules +- HomelabPrometheusTargetDown: up == 0 for 5m, severity warning. +- HomelabSmartDiskHealthFailed: homelab_smartctl_health_passed == 0 for 10m, severity critical. +- HomelabRootFilesystemLowSpace: root filesystem available below 15% for 15m, severity warning. +- HomelabPrometheusConfigReloadFailed: prometheus_config_last_reload_successful == 0 for 5m, severity critical. +- HomelabP0HealthStatusNotOk: homelab_health_status tier P0 equals 0 for 15m. +- HomelabP0BackupRestoreHealthStale: P0 backup/offhost/restore health older than 36h. +- HomelabP0WeeklyEdgeVmVzdumpHealthStale: weekly edge-vm vzdump health older than 8d. +- HomelabP1BackupHealthStale: P1 backup health older than 7d. +- HomelabMailCloudRestoreDrillStale: mail-cloud restore drill older than 8d. +- HomelabMkdocsRuntimePublishStale: MkDocs runtime publish older than 30d. +- HomelabHealthMetricsCollectorMissing: homelab health metrics absent from Prometheus. +- HomelabExternalCanaryHealthNotOkOrStale: external canary not OK or older than 30m. +- HomelabBlackboxProbeFailed / Missing: blackbox probe failed or absent. +- HomelabVulnScanHealthNotOkOrStale: vulnerability scan failed or older than 8d. +- HomelabNpmplusCertExpiryHealthNotOkOrStale: NPMplus cert expiry health failed or older than 48h. +- HomelabBlackboxTlsCertificateExpiringSoon: TLS certificate expiry within 14 days. + +### Loki / Alloy log pipeline +- Loki version observed: grafana/loki:3.7.2. +- Alloy version observed: grafana/alloy:v1.17.0. +- Loki auth_enabled: false. +- Loki storage: local filesystem under /loki. +- Loki schema: tsdb v13. +- Loki retention_period: 14d. +- Alloy discovers Docker through socket-proxy at tcp://socket-proxy:2375. +- Alloy relabels container, compose_project, compose_service and host=edge-vm. +- Alloy forwards Docker logs to http://loki:3100/loki/api/v1/push. +- Loki readiness observed as "ready". + +### Runtime dashboard semantics +- backup-restore-dashboard.txt is an authoritative runtime dashboard file. +- Current observed runtime dashboard: STATUS=OK, ITEMS=69, NOT_OK=0. +- backup-restore-dashboard.json confirmed not_ok=[]. +- restore-drill-index.txt observed: STATUS=OK, RESTORE_PROOFS=16/16, MISSING empty. +- cluster-daily-status.txt observed: STATUS=OK, PROBLEM_COUNT=0. +- external-canary.txt observed: STATUS=OK, BAD=0. +- external canary checks include nc.gram1.ru, git.gram1.ru, auth.gram1.ru, backup.gram1.ru. +- Alertmanager local API is reachable. +- Gotify local /health returns green. +- ntfy local /v1/health returns healthy true. +- Uptime Kuma local endpoint returns HTTP 302 to /dashboard, which is expected for UI auth/redirect. +- Healthchecks local status endpoint returns HTTP 301, which is treated as reachable. + +### Certificate / vulnerability health +- npmplus-cert-expiry-health.txt is the standardized health alias for certificate expiry. +- npmplus-certificate-expiry.txt is the detailed cert expiry file. +- Detailed cert expiry file observed cert_count=9 and min_cert_days_left=75 with warn threshold 30. +- cluster-daily-status local extra check reports npmplus-cert-expiry status OK, problems=0, min_days=75. +- trivy-scan.txt observed: STATUS=OK, IMAGE_TOTAL=58, IMAGE_SCAN_OK=58, IMAGE_SCAN_ERROR=0, CRITICAL_TOTAL=112, HIGH_TOTAL=1650. +- vuln-scan.txt observed: STATUS=OK, MODE=full, IMAGES=58, TRIVY_OK=49, CRITICAL_MATCHES=203. + +### SLO backlog semantics +- Runtime dashboard OK does not mean SLO backlog is closed. +- slo-coverage-backlog-index.txt observed: STATUS=OK, BACKLOG_WARN_COUNT=12, RUNTIME_BLOCKER=NO. +- backup-restore-coverage-slo.txt observed: STATUS=WARN, APP_TOTAL=43, GREEN_COUNT=3, YELLOW_COUNT=21, RED_COUNT=19, MISSING_BACKUP_SIGNAL=31, MISSING_RESTORE_SIGNAL=28, MISSING_TIMER=31, MISSING_EXTERNAL_VALIDATION=43. +- Other observed WARN backlog files include container-image-lifecycle-slo, cron-job-monitoring-slo, direct-heartbeat-pilot-readiness, healthchecks-heartbeat-coverage, healthchecks-job-coverage, healthchecks-job-monitoring-slo, homepage-service-catalog-slo, ingress-dns-route-slo, observability-health-surface-slo and security-vulnerability-slo. +- SLO WARN files are tracked separately and are not runtime blockers unless explicitly promoted to P0 runtime checks. + +### Important file locations +- Main health dir on edge-vm: /var/lib/homelab-health. +- Prometheus config: /opt/stacks/observability-lite/prometheus/prometheus.yml. +- Prometheus rules: /opt/stacks/observability-lite/prometheus/rules/. +- Alertmanager config: /opt/stacks/observability-lite/alertmanager/alertmanager.yml. +- Loki config: /opt/stacks/loki-alloy/loki-config.yaml. +- Alloy config: /opt/stacks/loki-alloy/config.alloy. +- Blackbox exporter config: /opt/stacks/blackbox-exporter/blackbox.yml. +- Uptime Kuma DB: /opt/uptime-kuma-compose/data/kuma.db. +- Gotify DB: /opt/gotify-compose/data/gotify.db. +- ntfy DB/cache: /opt/stacks/ntfy/data/user.db and /opt/stacks/ntfy/cache/cache.db. +- Healthchecks settings/data: /opt/stacks/healthchecks/local_settings.py and /var/lib/healthchecks. + +### Known caveats +- A previous compact check looked for npmplus-cert-expiry.txt; use npmplus-certificate-expiry.txt for detailed cert expiry and npmplus-cert-expiry-health.txt for standardized health. +- health-metrics.txt is not required to exist as a health file; health metrics may be emitted via node-exporter textfile collector. +- Prometheus targets must be checked with explicit HTTP code and parsed JSON output, not a silent curl pipeline. + +### Proof +- Raw monitoring inventory: 143_MONITORING_ALERTING_HEALTH_RAW_INVENTORY.txt. +- Runtime/SLO/alerting status: 144_MONITORING_RUNTIME_SLO_ALERTING_STATUS.txt. +- Kuma/Gotify/ntfy/Healthchecks status: 145_KUMA_GOTIFY_NTFY_HEALTHCHECKS_SAFE_STATUS.txt. +- Corrected monitoring compact status: 146_MONITORING_COMPACT_STATUS_CORRECTED.txt. + +## PROMETHEUS_STATUS_CORRECTION_20260630 + +- Initial monitoring compact check after reference creation reported Prometheus API HTTP_CODE=000 on 127.0.0.1:9090. +- This was recorded as error-register item 30 and checked with proof files 149_PROMETHEUS_DIAG_AFTER_MONITORING_REFERENCE.txt, 150_PROMETHEUS_CONDITIONAL_RECOVERY.txt and 151_PROMETHEUS_REFERENCE_CORRECTION_PROOF.txt. +- The Monitoring/Alerting section must be interpreted together with the latest Prometheus correction proof, not only with the earlier secret-scan proof. + +## COMMAND_PREFLIGHT_RULE_20260630 + +- Strict operator rule: before every command, check both the error register and this reference file. +- Required visible markers before main action: ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK. +- If either check fails, do not run the main action. +- Do not use exit 1 in interactive SSH sessions. +- Proof: 152_COMMAND_PREFLIGHT_RULE_RECORDED.txt. + +## PROMETHEUS_TARGETS_SETTLED_20260630 + +- Prometheus was absent after the initial monitoring reference proof and was restarted in 150_PROMETHEUS_CONDITIONAL_RECOVERY.txt. +- Prometheus readiness proof exists in 151_PROMETHEUS_REFERENCE_CORRECTION_PROOF.txt. +- Target health was rechecked with an additional settled scrape proof in 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt. +- Interpret Prometheus status from the latest settled proof, not from the earlier immediate-after-start unknown target state. +## PROMETHEUS_TARGETS_SETTLED_PROOF_CORRECTION_20260630 + +- 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt is invalid and must not be used as evidence. +- Reason: nested Python inside SSH lost quoting and produced SyntaxError. +- Error-register item 32 records this mistake. +- Valid replacement proof: 154_PROMETHEUS_TARGETS_SETTLED_PROOF_NO_PYTHON.txt. +- Prometheus status must be interpreted from 154 and later proofs, not from 153. + +## MONITORING_PROMETHEUS_FINAL_CLOSED_20260630 + +- Prometheus was initially absent after monitoring reference creation. +- It was restarted and then verified after scrape settling. +- Valid settled proof: 154_PROMETHEUS_TARGETS_SETTLED_PROOF_NO_PYTHON.txt. +- Final observed state: HTTP_CODE=200, PROM_TARGETS_TOTAL=14, PROM_UP_COUNT=14, PROM_NON_UP_COUNT=0. +- Correction/secret scan proof: 156_PROMETHEUS_CORRECTION_PROOF_AND_SECRET_SCAN.txt. +- Invalid proof 153 must not be used. +## SECURITY_ACCESS_SECRETS_OPERATING_MODEL_20260630 + +### Command safety +- Before every command, check /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md and current 31_HOMELAB_REFERENCE.md. +- Print ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK before the main action. +- Do not run the main action if either check fails. +- Do not use exit 1 in interactive SSH sessions. +- Avoid long nested SSH/Python/PHP/SQL quoting chains. +- Do not print secrets. + +### Access model +- Primary operator entrypoint: root@pve01 / [PRIVATE_IP]. +- pve02 and pve03 are reached as root from pve01. +- edge-vm is reached as debian@[PRIVATE_IP] with sudo. +- forum-prod is reached through pve02 using [SENSITIVE_PATH] to root@[PRIVATE_IP]. +- PVE users observed: root@pam and prometheus@pve. +- prometheus@pve has PVEAuditor on / for Proxmox exporter access. + +### SSH and permissions +- pve01 root keys observed: id_rsa, id_ed25519 and public keys. +- pve02 root keys observed: id_rsa and forum-prod-ci-key. +- pve03 root key observed: id_rsa. +- edge-vm root key observed: id_ed25519; debian authorized_keys observed. +- Security finding: pve01/pve02/pve03 root authorized_keys files were observed as mode 777. +- Correction proof: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt. +- Private key material must never be copied into the reference. + +### Secret storage +- Primary private storage root: /var/lib/homelab-private. +- Edge private secrets root: /var/lib/homelab-private/secrets. +- NPMplus DB: /opt/npmplus/npmplus/database.sqlite, root-only mode observed. +- Gotify DB: /opt/gotify-compose/data/gotify.db. +- Uptime Kuma DB: /opt/uptime-kuma-compose/data/kuma.db. +- Vaultwarden DB: /opt/vaultwarden-compose/data/db.sqlite3. +- Rclone configs exist under root config paths and must not be printed. +- Env/secret inventory is path-only: owner, mode, size and path only. + +### Network exposure +- PVE nodes expose SSH :22, Proxmox :8006 and node-exporter :9100. +- pve01/pve02 expose homelab-health-http :9101. +- pve01 exposes NUT :3493 on localhost and [PRIVATE_IP]. +- edge-vm exposes public :80/:443 through NPMplus. +- edge-vm NPMplus admin :81 and socket-proxy :2375 are bound to 127.0.0.1. +- edge-vm registry cache :5000 is bound to [PRIVATE_IP]. +- edge-vm Home Assistant :8123 is intentionally LAN-exposed. +- Edge ingress hardening proof reports STATUS=OK. + +### Rotation and scan proofs +- Cloudflare token rotation completed without printing token values. +- Old Cloudflare token revocation was externally confirmed. +- XenForo SMTP password rotation completed without printing password values. +- XenForo SMTP auth was verified with STARTTLS-safe method. +- Current reference strict scan shows zero strict secret hits. +- Selected generated reference blocks show zero strict secret hits. + +### Proof +- Raw security/access inventory: 158_SECURITY_ACCESS_SECRETS_RAW_INVENTORY.txt. +- Secret scan and rotation proof index: 159_SECURITY_SECRET_SCAN_AND_ROTATION_PROOF_INDEX.txt. +- SSH permission correction proof: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt. +## SECURITY_SSH_PERMISSION_CORRECTION_20260630 + +- Initial proof 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt is not sufficient as closure evidence. +- Reason: stat without -L still showed 777 after chmod, which may be symlink mode rather than target file mode. +- Error-register item 35 records this proof-quality issue. +- Authoritative replacement proof: 163_PVE_ROOT_AUTHORIZED_KEYS_SYMLINK_AWARE_FIX.txt. +- Valid closure condition: target permissions checked with stat -L are 600 for authorized_keys files and [SENSITIVE_PATH] is 700. +- Final observed status in replacement proof: STATUS=SSH_AUTHORIZED_KEYS_TARGET_PERMISSIONS_OK. +## STORAGE_DISK_SMART_CAPACITY_REFERENCE_20260630 + +### Proxmox storage model +- Cluster storage is defined in /etc/pve/storage.cfg. +- local storage: dir /var/lib/vz, content iso,vztmpl,backup,import. +- local-lvm storage: lvmthin data, content rootdir,images. +- Nightly vzdump job writes to local storage. +- VM/CT images are primarily stored on local-lvm. +- Staging/offhost paths are under /mnt/staging where present. + +### Node disks and capacity +- pve01: primary disk previously inventoried as Lexar SSD NQ7A1 1TB. +- pve02: primary disk previously inventoried as SK800-1TB. +- pve03: primary disk previously inventoried as Samsung 870 EVO 500GB. +- pve01 storage usage was previously observed around local 32.56% and local-lvm 17.50%. +- pve02 storage usage was previously observed around local 12.86% and local-lvm 11.65%. +- pve03 storage usage was previously observed around local 35.79% and local-lvm 64.84%. +- Raw current df/lsblk/LVM/SMART evidence is in 166_STORAGE_DISK_SMART_CAPACITY_RAW_INVENTORY.txt. + +### Edge VM storage model +- edge-vm runs Docker application stacks. +- Docker app data lives mainly under /opt/stacks, /opt/npmplus, /opt/gotify-compose, /opt/uptime-kuma-compose, /opt/vaultwarden-compose and /var/lib application paths. +- Immich media is mounted separately at /mnt/immich-media in the container mapping. +- Docker volume and image usage is captured in docker system df output. +- Disk retention health is tracked by disk-retention-policy and disk-retention-cleanup files. + +### SMART and health monitoring +- PVE nodes run smartctl textfile timers. +- pve01/pve02/pve03 expose node-exporter :9100 for Prometheus. +- pve01 and pve02 expose homelab-health-http :9101. +- SMART/disk/capacity health evidence is collected from /var/lib/homelab-health where present. +- Prometheus alert rule HomelabSmartDiskHealthFailed watches homelab_smartctl_health_passed == 0. +- Prometheus alert rule HomelabRootFilesystemLowSpace watches root filesystem available below 15%. +- pve03 did not have /var/lib/homelab-health in previous backup-health collection, but it has node-exporter and smartctl timers. + +### Retention and cleanup +- Edge disk retention policy previously observed STATUS=OK. +- Edge disk retention policy observed ROOT_USED_PCT=70, REMOVED_DIRS=0, DOCKER_VOLUME_PRUNE=NO. +- Retention dry-run policy is designed to avoid destructive production changes. +- Broad Docker prune is not used as a default cleanup mechanism. +- Cleanup and retention actions must have explicit proof files. + +### Operational rules +- Before deleting or pruning storage, create or identify rollback/backup proof. +- Do not delete vzdump/offhost/cloud backup artifacts unless retention policy and proof explicitly allow it. +- Do not infer offhost success from a failed rsync. +- For backup artifacts, verify exact file, size, checksum or zstd/vma integrity where applicable. +- For SMART, use health output plus selected attributes, not full raw dumps with unnecessary noise. + +### Proof +- Raw storage/disk/SMART/capacity inventory: 166_STORAGE_DISK_SMART_CAPACITY_RAW_INVENTORY.txt. +- Earlier Proxmox node/storage inventory: 120_PROXMOX_CLUSTER_NODE_STORAGE_NETWORK_INVENTORY.txt. +- Backup/restore catalog: 141_REFERENCE_BACKUP_RESTORE_OFFHOST_CATALOG_BLOCK.txt. +- Monitoring rules and SMART alert model: 147_REFERENCE_MONITORING_ALERTING_HEALTH_BLOCK.txt. +## STORAGE_CAPACITY_CORRECTION_20260630 + +- Storage block integrity review proof: 169_STORAGE_BLOCK_INTEGRITY_AND_PVE03_CAPACITY_REVIEW.txt. +- 167_REFERENCE_STORAGE_DISK_SMART_CAPACITY_BLOCK.txt integrity status: OK. +- pve03 /mnt/staging current observed use percent: 77. +- pve03 staging is close to the existing WARN threshold of 80% if observed use is 75% or higher. +- pve01 disk-space.txt coverage visibly listed pve01_staging, pve02_staging and edge_immich_media; pve03_staging coverage must be added or explicitly documented elsewhere. +- Storage layer is operationally OK only if SMART remains PASSED and pve03 staging is tracked before it crosses WARN/CRIT thresholds. +- This is a capacity coverage note, not a secret or runtime outage. +## PVE03_STAGING_CAPACITY_MONITOR_20260630 + +- pve03 /mnt/staging was observed at 77% usage. +- Existing pve01 disk-space.txt did not visibly include pve03_staging coverage. +- A dedicated pve03 staging capacity health check was added on pve01. +- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt. +- Timer: homelab-pve03-staging-capacity-health.timer. +- Service: homelab-pve03-staging-capacity-health.service. +- Current observed status: OK. +- WARN threshold: 80%. +- CRIT threshold: 90%. +- Proof: 173_PVE03_STAGING_CAPACITY_HEALTH_INSTALLED.txt. + +## STORAGE_LAYER_FINAL_CLOSED_WITH_PVE03_MONITOR_20260630 + +- Storage/disk/SMART/capacity layer is closed with an explicit pve03 staging capacity monitor. +- pve03 /mnt/staging was near WARN threshold at 77%, so a dedicated health file and timer were added. +- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt. +- Timer: homelab-pve03-staging-capacity-health.timer. +- Final closure proof: 176_STORAGE_FINAL_CLOSURE_WITH_PVE03_MONITOR.txt. +## SERVICE_DEPENDENCY_MAP_20260630 + +### Ingress and DNS chain +- Home DHCP DNS: [PRIVATE_IP] and [PRIVATE_IP]. +- dns1: CT110 / [PRIVATE_IP] / AdGuard Home. +- dns2: CT111 / [PRIVATE_IP] / AdGuard Home. +- unbound1: CT112 / [PRIVATE_IP]:5335. +- unbound2: CT113 / [PRIVATE_IP]:5335. +- dns1 upstream: [PRIVATE_IP]:5335. +- dns2 upstream: [PRIVATE_IP]:5335. +- Public ingress HTTP/HTTPS: router forwards TCP 80/443 to edge-vm [PRIVATE_IP]. +- Reverse proxy: NPMplus on edge-vm, container npmplus, host networking, admin bound to 127.0.0.1:81. +- Edge runtime host: VM130 edge-vm, [PRIVATE_IP], Docker Compose projects count 43. +- NPMplus proxy routes count: 47. +- NPMplus certificates and proxy route source proof: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt. + +### Core public routes +| Route | Upstream | Runtime owner | Backup/health evidence | +|---|---|---|---| +| nc.gram1.ru | http://[PRIVATE_IP]:11000 | VM150 Nextcloud AIO | VM150 vzdump, nextcloud restore proof, external canary | +| uptime.gram1.ru | http://127.0.0.1:3001 | edge-vm / uptime-kuma | npmplus-kuma backup/restore, Kuma health | +| gotify.gram1.ru | http://127.0.0.1:8082 | edge-vm / gotify | gotify health, alert-routing proof | +| vault.gram1.ru | http://127.0.0.1:8083 | edge-vm / vaultwarden | vaultwarden backup/offhost/restore | +| dockge.gram1.ru | http://127.0.0.1:5001 | edge-vm / dockge | stack inventory | +| home.gram1.ru | http://127.0.0.1:3000 | edge-vm / homepage | homepage container, dashboard route | +| git.gram1.ru | http://127.0.0.1:3002 | edge-vm / gitea | gitea backup/offhost/restore | +| dozzle.gram1.ru | http://127.0.0.1:9999 | edge-vm / dozzle | docker stack inventory | +| paper.gram1.ru | http://127.0.0.1:8010 | edge-vm / paperless | paperless backup/offhost/restore | +| memos.gram1.ru | http://127.0.0.1:5230 | edge-vm / memos | memos backup/offhost/restore | +| photos.gram1.ru | http://127.0.0.1:2283 | edge-vm / immich | immich media/full consistency proof | +| auth.gram1.ru | http://127.0.0.1:9000 | edge-vm / authentik | authentik backup/offhost/restore | +| backup.gram1.ru | http://[PRIVATE_IP]:9101 | pve01 / homelab-health-http | backup dashboard and health proofs | + +### VPN routes on wildcard certificate +| Route | Upstream | Runtime owner | +|---|---|---| +| kuma.vpn.gram1.ru | http://127.0.0.1:3001 | uptime-kuma | +| homepage.vpn.gram1.ru | http://127.0.0.1:3000 | homepage | +| gitea.vpn.gram1.ru | http://127.0.0.1:3002 | gitea | +| filebrowser.vpn.gram1.ru | http://127.0.0.1:8085 | filebrowser | +| auth.vpn.gram1.ru | http://127.0.0.1:9000 | authentik | +| actual-budget.vpn.gram1.ru | http://127.0.0.1:5006 | actual-budget | +| grafana.vpn.gram1.ru | http://127.0.0.1:3003 | grafana | +| homebox.vpn.gram1.ru | http://127.0.0.1:7745 | homebox | +| mealie.vpn.gram1.ru | http://127.0.0.1:9001 | mealie | +| n8n.vpn.gram1.ru | http://127.0.0.1:5678 | n8n | +| netbox.vpn.gram1.ru | http://127.0.0.1:8000 | netbox | +| node-red.vpn.gram1.ru | http://127.0.0.1:1880 | node-red | +| beszel.vpn.gram1.ru | http://127.0.0.1:8090 | beszel | +| it-tools.vpn.gram1.ru | http://127.0.0.1:8084 | it-tools | +| karakeep.vpn.gram1.ru | http://127.0.0.1:3030 | karakeep | +| linkding.vpn.gram1.ru | http://127.0.0.1:9091 | linkding | +| minio.vpn.gram1.ru | http://127.0.0.1:9003 | minio | +| ntfy.vpn.gram1.ru | http://127.0.0.1:8055 | ntfy | +| searxng.vpn.gram1.ru | http://127.0.0.1:8888 | searxng | +| syncthing.vpn.gram1.ru | http://127.0.0.1:8384 | syncthing | +| alertmanager.vpn.gram1.ru | http://127.0.0.1:9093 | alertmanager | +| prometheus.vpn.gram1.ru | http://127.0.0.1:9090 | prometheus | +| homeassistant.vpn.gram1.ru | http://127.0.0.1:8123 | homeassistant | +| npmplus.vpn.gram1.ru | https://[PRIVATE_IP]:81 | NPMplus admin, localhost-bound on edge | +| proxmox.vpn.gram1.ru | https://[PRIVATE_IP]:8006 | pve01 Proxmox UI | +| dns1.vpn.gram1.ru | http://[PRIVATE_IP]:8080 | dns1 AdGuard | +| dns2.vpn.gram1.ru | http://[PRIVATE_IP]:8080 | dns2 AdGuard | +| checks.vpn.gram1.ru | http://127.0.0.1:8015 | healthchecks | +| vikunja.vpn.gram1.ru | http://127.0.0.1:8016 | vikunja | +| bookstack.vpn.gram1.ru | http://127.0.0.1:8017 | bookstack | +| pdf.vpn.gram1.ru | http://127.0.0.1:8018 | stirling-pdf | +| jellyfin.vpn.gram1.ru | http://127.0.0.1:8019 | jellyfin | +| audiobooks.vpn.gram1.ru | http://127.0.0.1:8020 | audiobookshelf | +| books.vpn.gram1.ru | http://127.0.0.1:8021 | calibre-web | + +### Disabled / special routes +- npm.gram1.ru exists in NPMplus but was observed disabled. +- turn.gram1.ru is an AdGuard rewrite to [PRIVATE_IP] for Nextcloud Talk/TURN. +- Router forwards TCP/UDP 3478 to Nextcloud [PRIVATE_IP]. +- Router forwards TCP/UDP 51820 for WireGuard on the Home bridge, not to edge-vm. + +### Critical dependency rules +- If a public app is down, check in this order: DNS rewrite/upstream, NPMplus route/cert, upstream container/VM, app health file, backup/restore proof. +- If a VPN route is down, check NetBird first, then NPMplus wildcard cert, then local upstream. +- If DNS is broken, check dns1/dns2 AdGuard and unbound1/unbound2 before app containers. +- If NPMplus is broken, use backup/restore evidence from npmplus-kuma-config and NPMplus DB backup. +- If edge-vm is broken, VM130 full-image backup/offhost/restore proofs are authoritative. +- If pve03 storage approaches WARN/CRIT, use pve03-staging-capacity health file. + +### Proof +- Raw service dependency evidence index: 177_SERVICE_DEPENDENCY_MAP_RAW_EVIDENCE_INDEX.txt. +- NPMplus route/cert rows: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt. +- AdGuard/Unbound rewrites/upstreams: 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt. +- Edge Docker container normalized map: 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt. +- Proxmox VM/CT inventory: 121_PROXMOX_VM_CT_CONFIG_INVENTORY_REDACTED.txt. +- Backup catalog: 141_REFERENCE_BACKUP_RESTORE_OFFHOST_CATALOG_BLOCK.txt. +- Monitoring model: 147_REFERENCE_MONITORING_ALERTING_HEALTH_BLOCK.txt. + +## SERVICE_DEPENDENCY_MAP_FINAL_CLOSED_20260630 + +- Service Dependency Map layer is closed. +- Integrity proof: 180_SERVICE_DEPENDENCY_MAP_INTEGRITY_REVIEW.txt. +- Secret scan proof: 179_SERVICE_DEPENDENCY_MAP_REFERENCE_PROOF_AND_SECRET_SCAN.txt. +- Final closure proof: 181_SERVICE_DEPENDENCY_MAP_FINAL_CLOSURE.txt. +## RUNBOOKS_RECOVERY_PROCEDURES_20260630 + +### Universal operator preflight +- Before every command, check /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md and the current 31_HOMELAB_REFERENCE.md. +- Required visible markers: ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK. +- If preflight fails, do not run the main action. +- Do not use exit 1 in interactive SSH sessions. +- Do not paste full terminal transcripts back into shell. +- Do not print secrets. +- Prefer short proof-producing commands over long nested quoting chains. + +### First triage order +- Check current reference layer first. +- Check latest proof file named by the relevant reference layer. +- Check health files under /var/lib/homelab-health where applicable. +- Check runtime state only after understanding the owning node, VM, container and proxy route. +- Record every failed command or misleading proof in the error register before moving on. + +### Public application down +- Start with Service Dependency Map. +- Check DNS path: dns1/dns2 AdGuard, then unbound1/unbound2. +- Check NPMplus route and certificate using NPMplus DB/proxy proof. +- Check upstream app container or VM. +- Check app-specific health, backup and restore proof. +- Check external-canary and blackbox/Prometheus if route is public. +- Do not change DNS, certificates or proxy routes without DB backup and proof. + +### DNS failure +- dns1 is CT110 at [PRIVATE_IP]. +- dns2 is CT111 at [PRIVATE_IP]. +- unbound1 is CT112 at [PRIVATE_IP]:5335. +- unbound2 is CT113 at [PRIVATE_IP]:5335. +- AdGuard upstreams must point to local Unbound pair. +- Rewrites include public routes for git, dozzle, paper, memos, photos, auth and backup to edge-vm. +- turn.gram1.ru points to Nextcloud [PRIVATE_IP]. +- Use DNS/Ingress reference and proof files before editing configs. + +### Ingress / NPMplus failure +- Edge VM is VM130 at [PRIVATE_IP]. +- NPMplus listens publicly on 80/443 and admin is bound to 127.0.0.1:81. +- NPMplus DB is /opt/npmplus/npmplus/database.sqlite. +- Before modifying certificates or proxy rows, create a DB backup. +- Cloudflare token values must never be printed. +- Use npmplus-kuma-config backup/offhost/restore proofs for recovery. + +### Edge VM failure +- VM130 is the Docker runtime host. +- First check Proxmox VM state and pve03 storage. +- Then use VM130 full-image vzdump/offhost/restore proofs. +- Current authoritative edge-vm backup/offhost/restore evidence is in backup catalog and VM130 proofs. +- If restoring, verify exact archive, checksum and VMA/ZSTD integrity before booting replacement. + +### Proxmox / VM / CT restore +- Primary nightly job: homelab-nightly-all at 03:30, zstd, snapshot. +- Included VMIDs: 110,111,112,113,130,150,160. +- Use backup catalog for latest known backup/offhost/restore evidence. +- Do not infer offhost success from failed rsync. +- Verify exact artifact, size and checksum where available. +- For VM130 and VM160, use their dedicated closure proofs. + +### Monitoring / alerting failure +- Prometheus is on edge-vm at 127.0.0.1:9090. +- Alertmanager is on 127.0.0.1:9093. +- Loki is on 127.0.0.1:3100. +- Uptime Kuma is on 127.0.0.1:3001. +- Gotify is on 127.0.0.1:8082. +- ntfy is on 127.0.0.1:8055. +- Healthchecks is on 127.0.0.1:8015. +- Prometheus proof 153 is invalid and must not be used. +- Use proof 154 and final closure 157 for Prometheus settled target state. + +### Backup dashboard / SLO interpretation +- Runtime dashboard OK means current operational checks are green. +- SLO backlog WARN means coverage improvement backlog, not necessarily runtime outage. +- Do not merge SLO backlog WARN into runtime blockers unless explicitly promoted. +- Use slo-coverage-backlog-index for backlog status. + +### Storage / capacity event +- pve03 staging is monitored separately because it was observed at 77%. +- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt. +- WARN threshold: 80%. +- CRIT threshold: 90%. +- Timer: homelab-pve03-staging-capacity-health.timer. +- Before cleanup, check retention policy and backup/offhost proof. +- Do not use broad Docker prune by default. + +### Security / secret incident +- Stop printing values immediately. +- Identify whether the leak is value, file path, or false-positive text. +- Rotate affected token/password if a value was exposed. +- Create backup before DB/config mutation. +- Re-run strict secret scan after rotation. +- Record proof files and external revocation confirmation where applicable. +- Cloudflare token and XenForo SMTP rotations already have closure proofs. + +### Forum / CodeVipe incident +- forum-prod is VM160 at [PRIVATE_IP]. +- Access path is through pve02 using [SENSITIVE_PATH] +- XenForo path: /var/www/codevipe/public. +- SMTP host: mail.pvepro.ru on port 587 with STARTTLS semantics. +- Do not enable smtpSsl=true blindly for port 587. +- Do not use fragile nested PHP/base64 SMTP checkers. +- Use XenForo SMTP rotation and auth proof files for current mail state. + +### Final evidence rules +- Every remediation gets a numbered proof file. +- Every reference block gets a proof and secret scan file. +- Invalid proof files must be explicitly superseded, not silently ignored. +- Reference closure requires integrity scan, secret scan and required-heading checks. +## HOMELAB_REFERENCE_FINAL_OPERATOR_STATUS_20260630 + +### Status +- Current homelab reference is complete for the audited local infrastructure scope. +- Final quality precheck passed. +- Required headings are present. +- Bad terminal/log marker scan is clean. +- Strict secret scan is clean. +- This is a reference/operator-status closure, not an archive/export. + +### Closed layers +- Critical runtime tails closure. +- Proxmox cluster, node, storage and VM/CT inventory. +- DNS, ingress, NPMplus certificates and AdGuard/Unbound model. +- Edge Docker stacks and container map. +- Backup, restore, offhost and cloud backup catalog. +- Monitoring, alerting, health dashboard and Prometheus correction. +- Security, access and secrets operating model. +- Storage, disk, SMART and capacity model with pve03 staging monitor. +- Service Dependency Map. +- Runbooks and recovery procedures. + +### Important superseded/invalid proofs +- 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt is invalid and must not be used. +- 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt is not sufficient by itself because symlink mode showed 777. +- Use 154 for Prometheus settled targets and 163 for symlink-aware SSH target permissions. + +### Current authoritative final proof set +- 185_HOMELAB_REFERENCE_FINAL_QUALITY_SCAN_PRECHECK.txt. +- 184_RUNBOOKS_RECOVERY_REFERENCE_PROOF_AND_SECRET_SCAN.txt. +- 181_SERVICE_DEPENDENCY_MAP_FINAL_CLOSURE.txt. +- 176_STORAGE_FINAL_CLOSURE_WITH_PVE03_MONITOR.txt. +- 165_SECURITY_ACCESS_SECRETS_FINAL_PROOF_AND_SECRET_SCAN.txt. +- 157_MONITORING_PROMETHEUS_FINAL_CLOSURE.txt. +- 142_BACKUP_RESTORE_OFFHOST_REFERENCE_PROOF_AND_SECRET_SCAN.txt. +- 136_EDGE_DOCKER_STACKS_REFERENCE_PROOF_AND_SECRET_SCAN.txt. +- 132_DNS_INGRESS_REFERENCE_PROOF_AND_SECRET_SCAN.txt. + +### Operator rule +- Before every future command, continue checking both the error register and this reference file. +- Required visible markers remain ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK. +## HOMELAB_REFERENCE_DEEP_AUDIT_CLOSURE_20260630 + +### Deep audit result +- Error register and reference consistency audit passed. +- Final proof files unresolved-marker audit passed. +- Reference proof-link audit passed. +- All referenced proof files exist. +- No unresolved final REVIEW_REQUIRED, PROOF_MISSING, HEADING_MISSING, BAD_MARKER, STRICT_SECRET, RULE_CHECK_FAIL, SyntaxError or PERM_BAD markers remained in final proof set. +- Invalid/superseded proofs are explicitly documented and must not be used as authoritative closure evidence. + +### Authoritative deep audit proofs +- 189_DEEP_ERROR_REGISTER_REFERENCE_CONSISTENCY_AUDIT.txt. +- 190_DEEP_PROOF_FILES_UNRESOLVED_MARKER_AUDIT.txt. +- 191_DEEP_REFERENCE_PROOF_LINK_AUDIT.txt. + +### Scope statement +- This closes the current audited local homelab reference scope. +- External systems can still receive separate dedicated passports if the scope is expanded later. + +## NEXTCLOUD_VM150_MAIL_CLOUD_BACKUP_20260630 +- VM150 Nextcloud Mail-cloud backup is installed on pve01. +- Timer: homelab-mail-cloud-nextcloud-vm-upload.timer. +- Health file: /var/lib/homelab-health/mail-cloud-nextcloud-vm.txt. +- Current proof health: STATUS=OK, MODE=recurring-chunked, PARTS=14, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no. +- Proof: 192_NEXTCLOUD_VM150_MAIL_CLOUD_CHUNKED_PROOF.txt. + +## ROUTER_RECURRING_BACKUP_BLOCKER_20260630 +- Router recurring backup from pve01 is not installed yet. +- Reason: pve01 cannot reach router management ports 2222, 5080, 5083 or 2323 on [PRIVATE_IP]. +- Current router backup state remains reference/self-test based until router management access is allowed or startup-config is exported by another approved path. +- Do not run bash/POSIX automation against Netcraze CLI; use router CLI syntax only. +- Proof: 195_ROUTER_RECURRING_BACKUP_BLOCKED_BY_MGMT_ACL_PROOF.txt. + +## EDGE_GAP_RESTORE_DRYRUN_CLOSURE_20260630 +- Restore dry-run gaps closed for Actual Budget, Home Assistant, Karakeep, Linkding, Mealie, n8n and Observability config. +- Production data was not restored or modified; checks used temporary extraction, SHA verification, tar listing/extraction and SQLite integrity where applicable. +- Current observed restore health: all 7 STATUS=OK with RESTORE_ATTEMPTED=YES and SECRET_PRINTED=no. +- Proof: 197_EDGE_GAP_RESTORE_DRYRUN_CLOSURE_PROOF.txt. + +## P2_SMALL_STACKS_BACKUP_RESTORE_CLOSURE_20260630 +- P2 small-stack backup and restore dry-run is installed on edge-vm. +- Covered paths: Syncthing config, ntfy cache/data, MinIO data, Homebox data and Node-RED data. +- Timer: homelab-p2-small-stacks-backup-restore.timer. +- Current observed health: STATUS=OK, BACKUP_PATH_COUNT=6, RESTORE_ATTEMPTED=YES, SQLITE_INTEGRITY_OK=yes, SECRET_PRINTED=no. +- Proof: 199_P2_SMALL_STACKS_BACKUP_RESTORE_CLOSURE_PROOF.txt. + +## BACKUP_CLOUD_AND_RESTORE_FINAL_CLOSURE_20260630 +- Backup/cloud/restore improvement pass is closed with runtime dashboard STATUS=OK and NOT_OK=0. +- VM150 Nextcloud now has Mail-cloud chunked backup with download verification. +- Restore dry-run gaps closed for Actual Budget, Home Assistant, Karakeep, Linkding, Mealie, n8n and Observability config. +- P2 small stacks now have backup and restore dry-run coverage for Syncthing config, ntfy, MinIO, Homebox and Node-RED. +- Router recurring backup remains blocked by management ACL and is documented as blocker, not silently closed. +- Proof: 201_BACKUP_CLOUD_AND_RESTORE_FINAL_CLOSURE_PROOF.txt. + +## ROUTER_STARTUP_CONFIG_MAIL_CLOUD_MANUAL_20260630 +- Router startup-config manual backup is stored in Mail-cloud crypt remote. +- Source file content is not printed in proofs or reference. +- Health file: /var/lib/homelab-health/router-startup-config-mail-cloud.txt. +- Current observed health: STATUS=OK, UPLOAD_OK=yes, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no. +- Recurring router backup is still blocked until pve01 can reach router management ports. +- Proof: 203_ROUTER_STARTUP_CONFIG_MAIL_CLOUD_MANUAL_PROOF.txt. + +## ROUTER_RUNNING_CONFIG_MAIL_CLOUD_RECURRING_20260630 +- Router running-config recurring Mail-cloud backup is installed on pve01. +- Timer: homelab-router-running-config-mail-cloud.timer. +- Health file: /var/lib/homelab-health/router-running-config-mail-cloud.txt. +- Current observed health: STATUS=OK, SSH_RC=0, LOOKS_CONFIG=yes, UPLOAD_OK=yes, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no. +- Manual startup-config Mail-cloud backup also exists as separate proof. +- Proof: 205_ROUTER_RUNNING_CONFIG_MAIL_CLOUD_RECURRING_PROOF.txt. + +## POST_BACKUP_PASS_SLO_RECONCILIATION_20260630 +- Post backup/cloud/restore pass SLO reconciliation is closed. +- Runtime backup dashboard remains STATUS=OK with NOT_OK=0. +- Restore index remains STATUS=OK with RESTORE_PROOFS=16/16. +- VM150 Mail-cloud backup, router manual/recurring Mail-cloud backups, P2 small-stacks and 7 edge restore dry-runs are all confirmed OK. +- Existing backup-restore-coverage-slo.txt is historical model output from 20260625 and does not represent current runtime blocker state. +- Current reconciliation health: STATUS=OK, RUNTIME_BLOCKER=NO, SECRET_PRINTED=no. +- Proof: 208_POST_BACKUP_PASS_SLO_RECONCILIATION_PROOF.txt. + +## POST_BACKUP_PASS_ALERTING_20260630 +- Prometheus alerting for post-backup-pass health files is installed and loaded. +- Rule file: /opt/stacks/observability-lite/prometheus/rules/homelab-post-backup-pass-alerts.yml. +- Alerts: HomelabPostBackupPassHealthNotOk and HomelabPostBackupPassHealthStale. +- Covered checks include VM150 Mail-cloud, router startup/running config Mail-cloud, P2 small-stacks and seven restore dry-run health files. +- Current proof confirms rule validation, Prometheus API visibility and up targets. +- Proof: 210_POST_BACKUP_PASS_ALERTING_PROOF.txt. + +## POST_BACKUP_PASS_ALERTS_NOT_FIRING_20260630 +- Post-backup-pass Prometheus alert rules are loaded and currently not firing. +- Corrected proof uses direct PromQL ALERTS query for firing state, not grep over rule names. +- Alerts checked: HomelabPostBackupPassHealthNotOk and HomelabPostBackupPassHealthStale. +- Proof: 213_POST_BACKUP_PASS_ALERTS_NOT_FIRING_CORRECTED_PROOF.txt. + +## MAIL_CLOUD_CAPACITY_RETENTION_20260630 +- Mail-cloud capacity and backup directory inventory was checked after post-backup-pass closure. +- VM150 Nextcloud, router startup/running config, P0 critical and edge-vm backup directories are present on crypt remotes. +- Observed capacity is sufficient: largest used remote was pve03-mail-03-crypt around 65 GiB of 1 TiB. +- Retention shape observed: P0 critical has 10 visible dirs with keep 14, edge-vm has 3 visible dirs with keep 4, new VM150/router jobs have initial dirs. +- Proof: 216_MAIL_CLOUD_CAPACITY_RETENTION_PROOF.txt. + +## ROUTER_BACKUP_SECRET_PERMISSION_20260630 +- Router recurring backup uses a dedicated routerbackup credential file on pve01. +- Secret file path: /var/lib/homelab-private/secrets/routerbackup.pass. +- File content must never be printed; only mode/owner/size may be checked. +- Current observed permission target: root-owned mode 600. +- Proof: 218_ROUTER_BACKUP_SECRET_PERMISSION_PROOF.txt. + +## NEW_BACKUP_UNITS_SYSTEMD_INVENTORY_20260630 +- New backup/restore systemd units and timers were inventoried after post-backup-pass closure. +- pve01 units checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup. +- edge-vm units checked: P2 small-stacks backup/restore, backup dashboard and restore drill index. +- Proof records LoadState, ActiveState, UnitFileState and failed-unit counts for pve01 and edge-vm. +- Proof: 222_NEW_BACKUP_UNITS_SYSTEMD_INVENTORY_PROOF.txt. + +## POST_BACKUP_PASS_PROM_HEALTH_COVERAGE_20260630 +- Prometheus health coverage was checked for the post-backup-pass checks. +- Covered checks: VM150 Mail-cloud, router startup/running config, P2 small-stacks and seven restore dry-runs. +- Proof records health status and age from Prometheus without printing credential values. +- Proof: 224_POST_BACKUP_PASS_PROM_HEALTH_COVERAGE_PROOF.txt. + +## POST_BACKUP_PASS_AUDIT_INDEX_20260630 +- Audit proof manifest was generated for post-backup-pass evidence files 192-225. +- Manifest records file names, sha256 hashes, count and missing-number check. +- Proof: 226_POST_BACKUP_PASS_AUDIT_INDEX_PROOF.txt. + +## POST_BACKUP_PASS_EXTENDED_FINAL_SNAPSHOT_20260630 +- Extended post-backup-pass final snapshot was created after capacity, retention, unit inventory, Prometheus coverage and audit index checks. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-227. +- Proof: 228_POST_BACKUP_PASS_EXTENDED_FINAL_SNAPSHOT_PROOF.txt. + +## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_229_20260630 +- Final audit manifest was generated after the extended final snapshot. +- Manifest covers proof numbers 192-229, including snapshot and reference-scan proofs. +- Manifest records file names, sha256 hashes, count and missing-number check. +- Proof: 230_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_229_PROOF.txt. + +## NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_20260630 +- New backup/restore unit files and executable script paths were inventoried and hashed. +- pve01 services checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup. +- edge-vm services checked: P2 small-stacks backup/restore, backup dashboard and restore drill index. +- Proof records FragmentPath, ExecStart path, mode/owner/size and sha256 hashes without printing script contents. +- Proof: 232_NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_PROOF.txt. + +## NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_CORRECTED_20260630 +- Corrected integrity proof was generated because one edge-vm executable required sudo for sha256sum. +- Corrected proof records hashes for pve01 and edge-vm unit fragments and executable scripts without printing script contents. +- Proof: 234_NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_CORRECTED_PROOF.txt. + +## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_235_20260630 +- Final audit manifest was regenerated after corrected unit/script integrity proof. +- Manifest covers proof numbers 192-235, including corrected integrity proof 234/235. +- Manifest records file names, sha256 hashes, count and missing-number check. +- Proof: 236_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_235_PROOF.txt. + +## NEW_BACKUP_TIMERS_INTEGRITY_20260630 +- New backup/restore timer unit files were inventoried and hashed. +- pve01 timers checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup. +- edge-vm timers checked: P2 small-stacks backup/restore, backup dashboard and restore drill index. +- Proof records FragmentPath, mode/owner/size, sha256 hashes and next-elapse metadata without printing sensitive values. +- Proof: 238_NEW_BACKUP_TIMERS_INTEGRITY_PROOF.txt. + +## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_239_20260630 +- Final audit manifest was regenerated after timer integrity proof. +- Manifest covers proof numbers 192-239 and records file names, sha256 hashes, count and missing-number check. +- Proof: 240_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_239_PROOF.txt. + +## POST_BACKUP_PASS_FINAL_SNAPSHOT_AFTER_TIMERS_20260630 +- Final snapshot was created after timer integrity and final audit manifest 192-239. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-241. +- Proof: 242_POST_BACKUP_PASS_FINAL_SNAPSHOT_AFTER_TIMERS_PROOF.txt. + +## ASSISTANT_COMMAND_BATCHING_RULE_20260630 +- Operator preference: combine related homelab commands into fewer package-style runs to reduce slow repetitive manual work. +- Applies to proof generation, reference updates, validation scans, hash/index refreshes and snapshot steps when safe. +- Mandatory preflight and sensitive-output hygiene still take priority. + +## POST_BACKUP_PASS_AUDIT_INDEX_AFTER_BATCHING_RULE_192_245_20260630 +- Audit manifest was regenerated after assistant command batching rule was recorded. +- Manifest covers proof numbers 192-245 and records file names, sha256 hashes, count and missing-number check. +- Proof: 246_POST_BACKUP_PASS_AUDIT_INDEX_AFTER_BATCHING_RULE_192_245_PROOF.txt. + +## POST_BACKUP_PASS_SNAPSHOT_AFTER_BATCHING_RULE_20260630 +- Snapshot was created after assistant command batching rule and audit manifest 192-245. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-247. +- Proof: 248_POST_BACKUP_PASS_SNAPSHOT_AFTER_BATCHING_RULE_PROOF.txt. + +## ALERTMANAGER_ROUTING_AND_CONFIG_20260630 +- Alertmanager routing/config health was checked after post-backup-pass alert rules were installed. +- Proof records container presence, config validation, readiness endpoint and sanitized route/receiver metadata. +- Sensitive receiver values and URLs are intentionally not printed. +- Proof: 250_ALERTMANAGER_ROUTING_AND_CONFIG_PROOF.txt. + +## AUDIT_INDEX_192_251_20260630 +- Audit manifest was regenerated after Alertmanager routing/config proof. +- Manifest covers proof numbers 192-251 with count and missing-number check. +- Proof: 252_AUDIT_INDEX_192_251_PROOF.txt. + +## RCLONE_CONFIG_PERMISSION_AND_REMOTE_INVENTORY_20260630 +- rclone config permissions and crypt remote inventory were checked without printing config contents. +- Proof records config owner/mode/size, remote names and capacity summaries for active Mail-cloud crypt remotes. +- Proof: 254_RCLONE_CONFIG_PERMISSION_AND_REMOTE_INVENTORY_PROOF.txt. + +## AUDIT_INDEX_192_255_20260630 +- Audit manifest was regenerated after rclone config/remote inventory proof. +- Manifest covers proof numbers 192-255 with count and missing-number check. +- Proof: 256_AUDIT_INDEX_192_255_PROOF.txt. + +## CURRENT_OPERATIONAL_ROLLUP_20260630 +- Current operational rollup was captured after rclone config/remote inventory proof. +- Rollup records failed-unit counts, next timers, Mail-cloud capacity, Prometheus target count, post-backup health status/age, post-backup firing alerts and Alertmanager readiness. +- Proof: 258_CURRENT_OPERATIONAL_ROLLUP_PROOF.txt. + +## AUDIT_INDEX_192_259_20260630 +- Audit manifest was regenerated after current operational rollup. +- Manifest covers proof numbers 192-259 with count and missing-number check. +- Proof: 260_AUDIT_INDEX_192_259_PROOF.txt. + +## SNAPSHOT_AFTER_CURRENT_OPERATIONAL_ROLLUP_20260630 +- Snapshot was created after current operational rollup and audit index 192-259. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-261. +- Proof: 262_SNAPSHOT_AFTER_CURRENT_OPERATIONAL_ROLLUP_PROOF.txt. + +## AUDIT_INDEX_192_263_20260630 +- Audit manifest was regenerated after snapshot following current operational rollup. +- Manifest covers proof numbers 192-263 with count and missing-number check. +- Proof: 264_AUDIT_INDEX_192_263_PROOF.txt. + +## NEW_BACKUP_SERVICE_RUNTIME_RESULT_20260630 +- Runtime result metadata was captured for new backup/restore service units. +- pve01 services checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup. +- edge-vm services checked: P2 small-stacks backup/restore, backup dashboard and restore drill index. +- Proof records LoadState, ActiveState, Result, ExecMainStatus and restart counters without printing sensitive values. +- Proof: 266_NEW_BACKUP_SERVICE_RUNTIME_RESULT_PROOF.txt. + +## AUDIT_INDEX_192_267_20260630 +- Audit manifest was regenerated after new backup service runtime-result proof. +- Manifest covers proof numbers 192-267 with count and missing-number check. +- Proof: 268_AUDIT_INDEX_192_267_PROOF.txt. + +## NEW_BACKUP_UNITS_JOURNAL_ERROR_SCAN_20260630 +- Journal warning/error counters were captured for new backup/restore units and timers over the last 24 hours. +- Proof records only counts, not journal message bodies, to avoid sensitive-output risk. +- Proof: 270_NEW_BACKUP_UNITS_JOURNAL_ERROR_SCAN_PROOF.txt. + +## AUDIT_INDEX_192_271_20260630 +- Audit manifest was regenerated after journal error scan proof. +- Manifest covers proof numbers 192-271 with count and missing-number check. +- Proof: 272_AUDIT_INDEX_192_271_PROOF.txt. + +## NEW_BACKUP_UNITS_JOURNAL_ERROR_TRIAGE_20260630 +- Journal warning/error triage was captured after warning/error counters were non-zero. +- Proof records per-unit warning/error counts and redacted journal fragments. +- Proof: 274_NEW_BACKUP_UNITS_JOURNAL_ERROR_TRIAGE_PROOF.txt. + +## AUDIT_INDEX_192_275_20260630 +- Audit manifest was regenerated after journal error triage proof. +- Manifest covers proof numbers 192-275 with count and missing-number check. +- Proof: 276_AUDIT_INDEX_192_275_PROOF.txt. + +## VM150_MAIL_CLOUD_JOURNAL_NOISE_CLASSIFICATION_20260630 +- VM150 Mail-cloud upload journal warnings/errors were classified after journal triage found non-zero counters. +- Current service result, Prometheus health and firing-alert status are used to decide whether journal noise is blocking. +- Proof includes redacted journal indicators only, not secrets. +- Proof: 278_VM150_MAIL_CLOUD_JOURNAL_NOISE_CLASSIFICATION_PROOF.txt. + +## AUDIT_INDEX_192_279_20260630 +- Audit manifest was regenerated after VM150 Mail-cloud journal-noise classification. +- Manifest covers proof numbers 192-279 with count and missing-number check. +- Proof: 280_AUDIT_INDEX_192_279_PROOF.txt. + +## SNAPSHOT_AFTER_VM150_JOURNAL_NOISE_CLASSIFICATION_20260630 +- Snapshot was created after VM150 Mail-cloud journal noise was classified as non-blocking historical noise. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-281. +- Proof: 282_SNAPSHOT_AFTER_VM150_JOURNAL_NOISE_CLASSIFICATION_PROOF.txt. + +## AUDIT_INDEX_192_283_20260630 +- Audit manifest was regenerated after snapshot following VM150 journal-noise classification. +- Manifest covers proof numbers 192-283 with count and missing-number check. +- Proof: 284_AUDIT_INDEX_192_283_PROOF.txt. + +## POST_BACKUP_PASS_CLOSURE_SUMMARY_20260630 +- Post-backup-pass closure summary was generated after VM150 journal-noise classification. +- Summary checks backup SLO closure, alert rules, firing-alert status, Prometheus health coverage, failed-unit state, Alertmanager readiness, rclone config hygiene, journal-noise classification and audit index completeness. +- Proof: 286_POST_BACKUP_PASS_CLOSURE_SUMMARY_PROOF.txt. + +## SNAPSHOT_AFTER_POST_BACKUP_PASS_CLOSURE_SUMMARY_20260630 +- Snapshot was created after post-backup-pass closure summary. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-287. +- Proof: 288_SNAPSHOT_AFTER_POST_BACKUP_PASS_CLOSURE_SUMMARY_PROOF.txt. + +## AUDIT_INDEX_192_289_20260630 +- Audit manifest was regenerated after post-backup-pass closure summary snapshot. +- Manifest covers proof numbers 192-289 with count and missing-number check. +- Proof: 290_AUDIT_INDEX_192_289_PROOF.txt. + +## TOMORROW_FIRST_RUN_VERIFICATION_PLAN_20260630 +- First scheduled-run verification should be done after 2026-07-01 08:15 MSK. +- Verify P2 small-stacks, router running-config Mail-cloud and VM150 Nextcloud Mail-cloud first automatic runs. +- Also verify service results, health status/age, Mail-cloud directories, dashboard, restore index, Prometheus firing alerts and Alertmanager readiness. +- Proof: 292_TOMORROW_FIRST_RUN_VERIFICATION_PLAN_PROOF.txt. + +## AUDIT_INDEX_192_293_20260630 +- Audit manifest was regenerated after tomorrow first-run verification plan. +- Manifest covers proof numbers 192-293 with count and missing-number check. +- Proof: 294_AUDIT_INDEX_192_293_PROOF.txt. + +## END_OF_DAY_POST_BACKUP_PASS_SNAPSHOT_20260630 +- End-of-day snapshot was created after post-backup-pass closure summary and tomorrow first-run verification plan. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range before this snapshot: 192-295. +- Proof: 296_END_OF_DAY_POST_BACKUP_PASS_SNAPSHOT_PROOF.txt. + +## AUDIT_INDEX_192_297_20260630 +- Final end-of-day audit manifest was generated after post-backup-pass snapshot. +- Manifest covers proof numbers 192-297 with count and missing-number check. +- Proof: 298_AUDIT_INDEX_192_297_PROOF.txt. + +## HOME_PORTAL_DISCOVERY_20260630 +- Home portal discovery started for https://home.gram1.ru/. +- Goal: make the portal show and open all web services, including external service cards such as NetBird and mail. +- Discovery proof records DNS, HTTPS/TLS headers, reverse-proxy candidates, compose files, domain references and homepage config candidates without printing secrets. +- Proof: 300_HOME_PORTAL_DISCOVERY_PROOF.txt. + +## HOME_PORTAL_CONFIG_AND_SERVICE_INVENTORY_20260630 +- Home portal config and service inventory was collected for https://home.gram1.ru/. +- Inventory records portal containers, config files, redacted card lines, compose web hints, domain references and common subdomain probes. +- Proof: 302_HOME_PORTAL_CONFIG_AND_SERVICE_INVENTORY_PROOF.txt. + +## HOME_PORTAL_GAP_ANALYSIS_20260630 +- Home portal gap analysis was collected for gethomepage/homepage behind npmplus. +- Proof records Homepage config hashes, redacted current cards/bookmarks, current URLs, npmplus route lines, local web-port probes and known service container candidates. +- Proof: 304_HOME_PORTAL_GAP_ANALYSIS_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_305_20260630 +- Home portal audit manifest was generated for proof numbers 300-305. +- Proof: 306_HOME_PORTAL_AUDIT_INDEX_300_305_PROOF.txt. + +## HOME_PORTAL_CARD_AND_API_ERROR_ANALYSIS_20260630 +- Home portal card/API-error analysis was collected before editing Homepage. +- Proof records current card URLs/titles, redacted widget config, redacted Homepage log errors, NPMPlus internal route files/routes and candidate public/VPN cards. +- Proof: 308_HOME_PORTAL_CARD_AND_API_ERROR_ANALYSIS_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_309_20260630 +- Home portal audit manifest was regenerated after card/API-error analysis. +- Manifest covers proof numbers 300-309 with count and missing-number check. +- Proof: 310_HOME_PORTAL_AUDIT_INDEX_300_309_PROOF.txt. + +## HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_APPLY_20260630 +- Homepage services.yaml was backed up, duplicate VPN cards were removed where a non-VPN card existed, and missing web cards for NetBird, Mail and Webmail were added without VPN suffix in the card names. +- SMTP/IMAP were intentionally not added as cards because they are protocol endpoints, not web UI cards. +- Homepage container was restarted and portal/card URLs were checked. +- Proof: 312_HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_APPLY_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_313_20260630 +- Home portal audit manifest was regenerated after duplicate-card cleanup and external-card addition. +- Manifest covers proof numbers 300-313 with count and missing-number check. +- Proof: 314_HOME_PORTAL_AUDIT_INDEX_300_313_PROOF.txt. + +## HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_CORRECTED_APPLY_20260630 +- Corrected Homepage apply removed duplicate VPN cards where a non-VPN card existed and added NetBird, Mail and Webmail cards with non-VPN names. +- The added card URLs use currently reachable vpn.gram1.ru web endpoints because public netbird/mail/webmail hosts did not resolve/open during analysis. +- SMTP/IMAP were not added as cards because they are protocol endpoints, not web UIs. +- Proof: 316_HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_CORRECTED_APPLY_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_317_20260630 +- Home portal audit manifest was regenerated after corrected duplicate-card cleanup and external-card addition. +- Manifest covers proof numbers 300-317 with count and missing-number check. +- Proof: 318_HOME_PORTAL_AUDIT_INDEX_300_317_PROOF.txt. + +## HOMELAB_COMMAND_SAFETY_HARDENING_20260630 +- Command safety rule strengthened after the home portal base64 apply failure. +- Opaque generated payloads are not acceptable for homelab changes; commands must be transparent, bounded, and validated before modification. +- Success requires content-specific checks in addition to service/runtime checks. +- Proof: 320_HOMELAB_COMMAND_SAFETY_HARDENING_RULE_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_321_20260630 +- Home portal audit manifest was regenerated after command-safety hardening rule. +- Manifest covers proof numbers 300-321 with count and missing-number check. +- Proof: 322_HOME_PORTAL_AUDIT_INDEX_300_321_PROOF.txt. + +## HOME_PORTAL_POST_APPLY_VALIDATION_AND_API_TRIAGE_20260630 +- Home portal was validated after duplicate cleanup and external-card addition. +- Proof checks portal HTTP status, Homepage container state, required cards, duplicate VPN card pairs, current card URLs, redacted API-error logs and widget configuration. +- Proof: 324_HOME_PORTAL_POST_APPLY_VALIDATION_AND_API_TRIAGE_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_325_20260630 +- Home portal audit manifest was regenerated after post-apply validation and API triage. +- Manifest covers proof numbers 300-325 with count and missing-number check. +- Proof: 326_HOME_PORTAL_AUDIT_INDEX_300_325_PROOF.txt. + +## HOME_PORTAL_API_ERROR_ROOT_CAUSE_20260630 +- Homepage API error root-cause analysis was collected after the UI showed API errors. +- Current evidence points to the Open-Meteo/weather widget timing out from Homepage, separate from service cards. +- HTTP 200 for the portal is not sufficient for UI closure when Homepage logs still contain API errors. +- Proof: 328_HOME_PORTAL_API_ERROR_ROOT_CAUSE_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_329_20260630 +- Home portal audit manifest was regenerated after API-error root-cause analysis. +- Manifest covers proof numbers 300-329 with count and missing-number check. +- Proof: 330_HOME_PORTAL_AUDIT_INDEX_300_329_PROOF.txt. + +## HOME_PORTAL_DISABLE_OPENMETEO_WIDGET_20260630 +- Open-Meteo/weather widget was disabled after root-cause analysis showed Homepage API errors from api.open-meteo.com timeouts. +- This keeps the portal focused on service-launcher cards and prevents external weather API failures from causing UI API errors. +- Portal closure now requires current Homepage API-error logs to be zero after restart/reload. +- Proof: 332_HOME_PORTAL_DISABLE_OPENMETEO_WIDGET_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_333_20260630 +- Home portal audit manifest was regenerated after disabling Open-Meteo/weather widget. +- Manifest covers proof numbers 300-333 with count and missing-number check. +- Proof: 334_HOME_PORTAL_AUDIT_INDEX_300_333_PROOF.txt. + +## HOME_PORTAL_LINK_OPEN_AUDIT_20260630 +- Home portal card links were audited after Open-Meteo/weather widget was disabled. +- Audit extracts current Homepage href URLs and checks whether each opens with an acceptable HTTP status. +- Acceptable statuses include success, redirects and auth-gated 401/403 pages because those still indicate a reachable web service. +- Proof: 336_HOME_PORTAL_LINK_OPEN_AUDIT_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_337_20260630 +- Home portal audit manifest was regenerated after link-open audit. +- Manifest covers proof numbers 300-337 with count and missing-number check. +- Proof: 338_HOME_PORTAL_AUDIT_INDEX_300_337_PROOF.txt. + +## HOME_PORTAL_EXPLICIT_VPN_DUPLICATE_CLEANUP_CORRECTED_20260630 +- Explicit VPN duplicate cards were removed from Homepage with a short transparent perl edit: Homepage VPN, Authentik VPN, Gitea VPN and Uptime Kuma VPN. +- Old router URL [PRIVATE_IP] was replaced with [PRIVATE_IP] where present. +- Proof: 342_HOME_PORTAL_EXPLICIT_VPN_DUPLICATE_CLEANUP_CORRECTED_PROOF.txt. + +## HOME_PORTAL_REMAINING_DUPLICATE_OCCURRENCE_ANALYSIS_20260630 +- Remaining Homepage duplicate/VPN/router occurrences were inspected after corrected cleanup still reported REVIEW. +- Proof records exact redacted file/line occurrences before another edit. +- Proof: 344_HOME_PORTAL_REMAINING_DUPLICATE_OCCURRENCE_ANALYSIS_PROOF.txt. + +## HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RECHECK_20260630 +- Homepage duplicate cleanup was rechecked against active config files only, excluding logs and backup files. +- Previous broad grep included historical logs/backups and therefore overstated remaining active duplicates. +- Proof: 346_HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RECHECK_PROOF.txt. + +## HOME_PORTAL_ACTIVE_VPN_DUPLICATES_FINAL_CLEANUP_20260630 +- Active Homepage services/bookmarks files were cleaned from semantic VPN duplicate cards and old router URL references. +- Removed targets: Homepage VPN, Authentik VPN, Gitea VPN and Uptime Kuma VPN, plus matching vpn.gram1.ru duplicate URLs. +- Proof: 350_HOME_PORTAL_ACTIVE_VPN_DUPLICATES_FINAL_CLEANUP_PROOF.txt. + +## HOME_PORTAL_NPMPLUS_DEFAULT_ROUTE_ANALYSIS_20260630 +- Home portal NetBird/Mail/Webmail cards were investigated after user saw the NPMPlus default page from client [PRIVATE_IP]. +- Analysis checks URL body fingerprints, NPMPlus route config matches, service container candidates and Homepage card lines. +- Proof: 356_HOME_PORTAL_NPMPLUS_DEFAULT_ROUTE_ANALYSIS_PROOF.txt. + +## HOME_PORTAL_CURRENT_STATE_20260630 +- Active Homepage URL: https://home.gram1.ru. +- Homepage container is running and portal HTTP check returned 200 after latest changes. +- NetBird card points to https://nb.pvepro.ru. +- Mail card points to https://mail.pvepro.ru. +- Webmail card was removed; no separate Webmail card is currently configured. +- Router bookmark points to http://[PRIVATE_IP]:5080 for operator client LAN [PRIVATE_IP]. +- Homepage services.yaml was updated via YAML-aware edit for siteMonitor fields. +- Current active services.yaml has SITEMONITOR_COUNT=43. +- Latest validation before this reference update: YAML_ERRORS_LAST_5M=0, HOME_HTTP_CODE=200, HOMEPAGE_RUNNING=true. +- Do not use sed-only insertion for siteMonitor; use YAML-aware editing and validate YAML before restart. +- Related active-state checks: NB_LINK_ACTIVE=yes, MAIL_LINK_ACTIVE=yes, WEBMAIL_ACTIVE=no, ROUTER_192_168_1_1_ACTIVE=yes. + +## HOME_PORTAL_DIRECT_LINK_FIX_20260630 +- Direct Homepage link correction requested by operator. +- NetBird: https://nb.pvepro.ru. +- Mail: https://mail.pvepro.ru. +- Webmail card removed. +- Router restored to http://[PRIVATE_IP]:5080. + +## HOME_PORTAL_YAML_AWARE_SITEMONITOR_20260630 +- siteMonitor fields were added using YAML-aware PyYAML editing, not line-based sed insertion. +- Added siteMonitor to 43 service cards. +- Excluded cards: Homepage, NPMplus, Router and Public Domain. +- YAML validation passed before restart: YAML_VALIDATE_RC=0. +- Homepage restarted successfully and reported YAML_ERRORS=0. + +## HOMELAB_CLUSTER_BACKLOG_20260630 +- Cluster backlog was created at /etc/pve/HOMELAB_CLUSTER_BACKLOG.md. +- PBS is intentionally out of scope; backup strategy remains Mail-cloud based. +- First next action: finish Homepage final validation. + +## HOMEPAGE_BACKUP_GAP_CLASSIFICATION_20260630 +- Homepage backup coverage gap classification file created at /etc/pve/HOMEPAGE_BACKUP_GAP_CLASSIFICATION.md. +- It defines closure requirements for the 13 remaining Homepage backup coverage gap/special-review rows. +- PBS is intentionally out of scope; closure must use Mail-cloud backups or explicit stateless/config-only classification. + +## HOMEPAGE_EXISTING_BACKUP_EVIDENCE_CLOSURE_20260630 +- Homepage backup coverage matrix started closing existing-evidence rows. +- Router maps to existing Mail-cloud startup/running-config backup proofs when ROUTER_EXISTING_EVIDENCE=yes. +- MinIO Console maps to existing P2 small-stacks backup/restore proof when MINIO_EXISTING_EVIDENCE=yes. +- Proof: 364_HOMEPAGE_EXISTING_BACKUP_EVIDENCE_CLOSURE_PROOF.txt. + +## HOME_EXTERNAL_CLOUDFLARE_ALEXHOST_20260630 +- Homepage External group includes Cloudflare card: https://dash.cloudflare.com. +- Homepage External group includes AlexHost Billing card: https://bill.alexhost.com/. +- Both cards have siteMonitor enabled for green status dots. +- YAML validation passed, Homepage returned HTTP 200, container running, and no Homepage YAML errors were seen after restart. +- Proof: 368_HOME_EXTERNAL_CLOUDFLARE_ALEXHOST_PROOF.txt. + +## HOME_EXTERNAL_RELAY_REMOVED_20260630 +- Homepage External card/link for relay.pvepro.ru was removed because relay.pvepro.ru had DNS but no reachable HTTP/HTTPS endpoint from edge-vm. +- Proof: 382_HOME_EXTERNAL_RELAY_REMOVE_PROOF.txt. + +## ROUTER_CLI_PERMISSION_LIMIT_20260630 +- Netcraze routerbackup SSH access is read-only for backup: show running-config works, but ACL/config commands are denied. +- Homepage Moscow Router red-dot fix requires router ACL/Web UI/admin-level change, not routerbackup. +- Relevant proofs: 391, 393, 394, 395. + +## ROUTER_CLI_PROOF_REPAIR_20260630 +- Error register updated for blank proof summary extraction in 395. +- Corrected derived counts are recorded in 396_ROUTER_CLI_ERROR_REGISTER_REPAIR_PROOF.txt. + +## HOME_EXTERNAL_NETBIRD_ALEXHOST_NORMALIZE_20260630 +- Homepage External NetBird and AlexHost Billing href/siteMonitor URLs normalized with trailing slash. +- Homepage container node checks returned HTTP 200 for both URLs. +- Proof: 397_HOME_EXTERNAL_NETBIRD_ALEXHOST_NORMALIZE_PROOF.txt. + +## ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701 +- Moscow Router ACL is restored and correct after manual Web UI edits. +- Remaining blocker for Homepage green dot is HTTP/HTTPS 403 from router web service to edge-vm/Bridge1, not network reachability. +- Relevant proofs: 399, 400, 401, 402. + +## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_20260701 +- Moscow Router Homepage card href: http://[PRIVATE_IP]:5080. +- Moscow Router Homepage siteMonitor: http://[PRIVATE_IP]:18083/router-moscow. +- Health endpoint service on edge-vm: homelab-router-moscow-health-http.service. +- Endpoint semantics: returns 200 when TCP to router [PRIVATE_IP]:5080 succeeds, else 503. +- Relevant proofs: 405, 406, 407. + +## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_FIXED_20260701 +- Moscow Router Homepage card target after fixed apply: href http://[PRIVATE_IP]:5080, siteMonitor http://[PRIVATE_IP]:18083/router-moscow. +- Fixed updater handles Homepage service-name-key YAML shape and property-style YAML shape. +- Relevant proofs: 405, 406, 407, 408. + +## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_APPLIED_EXACT_20260701 +- Moscow Router Homepage card exact active YAML shape was service-key style. +- Applied href: http://[PRIVATE_IP]:5080. +- Applied siteMonitor: http://[PRIVATE_IP]:18083/router-moscow. +- Health endpoint service: homelab-router-moscow-health-http.service on edge-vm. +- Relevant proofs: 406, 409, 410. + +## FORUM_PROD_VM160_REBUILD_BASELINE_20260701 +- Old test VM160 was stopped and destroyed cleanly: proof 419, CONFIG_EXISTS=no, VM160_DISKS_LEFT=0. +- New VM160 forum-prod was recreated on pve02 with 4 vCPU, 8 GiB RAM, 200G disk, MAC BC:24:11:B6:45:ED, IP [PRIVATE_IP]/24, gateway [PRIVATE_IP]. +- New VM identity confirmed: forum-prod / forum-prod.gram1.ru, Debian 12 bookworm, SSH OK, qemu-agent OK, chrony OK. +- Swapfile 8G enabled with /swapfile mode 600 and fstab entry: proof 430. Proof 429 is invalid due nested command substitution quoting. +- SURY PHP repo added; PHP 8.5.7 available and installed with Nginx 1.22.1, MariaDB 10.11.14 and required XenForo modules. +- Beget-compatible profile applied: memory_limit 256M, post/upload 1024M, max_input_vars 10000, MariaDB utf8mb4/utf8mb4_unicode_ci, innodb_buffer_pool_size 2G. +- Operator reference copies placed in /root/homelab-operator inside VM160 for local VM command preflight. + +## FORUM_PROD_CODEVIPE_FIRST_RECIPE_STRATEGY_20260701 +- Current VM160 is now laboratory state, not final production closure. +- Bulk import of five forums reached DB/files/nginx/php-fpm ready state, but frontend body stayed empty under PHP 8.5.7. +- Strategy changed: prove CodeVipe first on controlled runtime, likely PHP 8.3 or PHP 8.2, then destroy VM160 and rebuild all five forums from scratch using the proven recipe. +- Do not publish Cloudflare/NPMplus routes for these forums until CodeVipe recipe is proven and final rebuild is complete. +- Proofs: failed/diagnostic chain 451-480; next snapshot proof 481 before further lab work. + +## FORUM_PROD_FRESH5_DEPLOYED_OK_20260701 +- VM160 forum-prod on pve02, IP [PRIVATE_IP]. +- Runtime: Debian 12, Nginx, MariaDB 10.11, PHP 8.3-FPM. +- Five XenForo forums deployed from fresh ZIP backups: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- Webroots: /var/www/forums/{codevipe,gamevipe,hkmods,zakrutim,dsmods}/public. +- DBs: xf_codevipe, xf_gamevipe, xf_hkmods, xf_zakrutim, xf_dsmods. +- Final health proof inside VM: /root/evidence/513_FORUM_PROD_FRESH5_FINAL_HEALTH_PROOF.txt. +- Final result: all five frontend/admin HTTP 200, www redirects 301, internal_data 403, no new XenForo errors. +- Lesson: fresh file ZIPs contain populated internal_data/code_cache; broken branch lacked compiled template cache. + +## OPERATOR_RULE_CONTINUE_WITH_NEXT_COMMAND_20260701 +- Rule: when an infrastructure step ends at an expected prompt or safe checkpoint, do not stop with only a status summary. +- Rule: immediately provide the next executable command in the same response. +- Applies to: SSH enter/exit points, VM prompt confirmations, snapshot confirmations, file copy confirmations, successful health checks, and other obvious next-step transitions. +- Exception: stop only if the previous output contains an error, ambiguity, dangerous state, missing file, secret exposure risk, or a required user decision. + +## OPERATOR_RULE_BATCH_2_3_COMMANDS_20260701 +- Rule: when safe, provide 2-3 sequential executable commands in one response instead of stopping after each expected checkpoint. +- Applies after: successful prompt transitions, clean health checks, completed copies, completed snapshots, and non-destructive inventory. +- Do not batch commands across: destructive operations, public exposure, DNS/SSL changes, secret handling, unclear errors, missing files, or required user decisions. +- Each command must still print ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK before work. + +## FORUM_PROD_FRESH5_PUBLIC_CUTOVER_OK_20260701 +- Public cutover completed for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- Cloudflare A records for main and www names point to edge public IP 95.84.154.183. +- Edge NPMplus manual routes terminate TLS with real Let’s Encrypt certificates and proxy to forum-prod VM160 at [PRIVATE_IP]:80. +- Final public proof: /root/evidence/546_PUBLIC_FORUMS_FINAL_DNS_HTTPS_PROOF.txt on pve01. +- Expected public state: main domains HTTPS 200 with ssl_verify_result 0; HTTP redirects to HTTPS; www redirects to main domain through forum-prod canonical vhosts. + +## EDGE_FORUM_PUBLICATION_BACKUP_20260701 +- Edge NPMplus forum publication backup created after public cutover. +- Backup location: /opt/npmplus/manual-backups/forum-public-ok-*.tar.gz on edge-vm [PRIVATE_IP]. +- Includes manual proxy_host configs 200-204 and Let’s Encrypt forum certificates under /opt/npmplus/tls/forum-certs. + +## FORUM_CERT_RENEWAL_CONFIGURED_20260701 +- Edge certificate renewal configured on edge-vm [PRIVATE_IP]. +- Token stored at /opt/npmplus/secure/forum_cf_token.env mode 600 root-only. +- Renewal script: /opt/npmplus/scripts/renew-forum-certs.sh. +- Cron: /etc/cron.d/forum-cert-renew, daily 03:17. +- Proof: /root/evidence/550_EDGE_FORUM_CERT_RENEWAL_SETUP_PROOF.txt on pve01. + +## CLOUDFLARE_FORUM_DNS_AUDIT_OK_20260701 +- Full Cloudflare DNS audit completed for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- Proof: /root/evidence/554_CLOUDFLARE_FULL_DNS_AUDIT_PROOF.txt on pve01. +- Result: root and www A records for all five zones point to 95.84.154.183. +- Result: no root/www AAAA records, no root/www CNAME conflicts, no records with old IP 87.236.18.45. +- Remaining cdn.* CNAME and TXT/DMARC/DKIM records were observed and not changed. + +## FORUMS_FINAL_DNS_AUDIT_SNAPSHOT_OK_20260701 +- Final post-DNS-audit snapshot completed after shortening Proxmox snapshot name. +- Snapshot name: forum-dns-ok-065203Z +- Proof: /root/evidence/557_FINAL_SHORT_SNAPSHOT_AFTER_DNS_AUDIT_PROOF.txt on pve01. +- Current public state: all five main domains resolve to 95.84.154.183 and return HTTPS 200 with valid certificates. + +## FORUM_LOCAL_BACKUP_CONFIGURED_20260701 +- Local forum backup configured inside VM160 forum-prod. +- Script: /root/scripts/forum-backup.sh. +- Cron: /etc/cron.d/forum-local-backup, daily 02:42. +- Retention: removes per-run directories older than 14 days; compressed archives remain under /var/backups/forums. +- Proof: /root/evidence/560_FORUM_BACKUP_SCRIPT_SETUP_PROOF.txt on pve01. + +## FORUM_MAIL_NOREPLY_AND_SPF_CLEANUP_20260701 +- XenForo email fields defaultEmailAddress/contactEmailAddress/bounceEmailAddress/emailSenderName set to noreply@pvepro.ru for all five forums. +- SPF TXT on codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru changed from stale 87.236.18.45 to v=spf1 -all. +- DKIM, DMARC and cdn.* CNAME records were not changed. +- Proofs: 564_UPDATE_XENFORO_EMAIL_NOREPLY_PROOF.txt, 565_CF_UPDATE_FORUM_SPF_NOREPLY_PROOF.txt, 566_FORUM_MAIL_NOREPLY_FINAL_AUDIT_PROOF.txt. + +## FORUM_CDN_RECORDS_DELETED_20260701 +- Deleted cdn.* CNAME records for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- Reason: old cdn.* records pointed to previous provider/trbcdn and no current public/local forum references were observed. +- Root and www A records remain on 95.84.154.183. +- Proofs: /root/evidence/567_CF_DELETE_FORUM_CDN_RECORDS_PROOF.txt and /root/evidence/568_CDN_DELETE_FINAL_AUDIT_REFERENCE_PROOF.txt on pve01. + +## FORUM_SPF_DEDUPED_AND_CDN_DELETED_20260701 +- Fixed duplicate SPF state caused by earlier failed cleanup attempt. +- Each of codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru now has exactly one SPF TXT: v=spf1 -all. +- Old SPF references to 87.236.18.* are absent. +- cdn.* CNAME records are absent. +- Public HTTPS remained healthy for all five forums. +- Proofs: 569_CF_FIX_DUPLICATE_SPF_RECORDS_PROOF.txt and 570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt. + +## FORUM_PUBLIC_DNS_MAIL_CLEAN_FINAL_20260701 +- Public web and mail-related DNS cleanup completed for five forum domains. +- @ and www A records point to 95.84.154.183. +- cdn.* CNAME records removed. +- SPF deduped to exactly one TXT: v=spf1 -all on each forum domain. +- Old provider IP 87.236.18.* absent from forum domain TXT records. +- XenForo visible email identity changed to noreply@pvepro.ru on all forums. +- Proofs: 570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt, 571_FORUM_MAIL_TRANSPORT_READINESS_PROOF.txt, 572_FORUM_PUBLIC_AND_MAIL_DNS_FINAL_SUMMARY_PROOF.txt. + +## FORUM_SMTP_TRANSPORT_PAUSED_20260701 +- Forum XenForo email identity is noreply@pvepro.ru, but actual SMTP transport is paused. +- Reason: failed SMTP authentication triggered Mailcow netfilter ban for 95.84.154.183. +- Broken msmtp secret/config files were removed from forum-prod. +- Do not re-enable SMTP until the mailbox/app password is rotated and tested with a single controlled attempt. +- Proof: /root/evidence/590_FORUM_SMTP_FAILED_CONFIG_DISABLED_PROOF.txt. + +## POST_INCIDENT_STABLE_STATE_20260701 +- Incident after failed forum SMTP testing resolved. +- Mailcow and NetBird are reachable again: mail.pvepro.ru/admin and nb.pvepro.ru return HTTP 200. +- Mailcow netfilter ban for 95.84.154.183 was removed on VPS mail.pvepro.ru. +- Forum sites remain healthy over HTTPS. +- Forum SMTP transport remains paused; broken msmtp configs/secrets removed from forum-prod. +- Do not retry SMTP until the mailbox/app password is rotated. +- Proofs: 589_POST_UNBAN_MAILCOW_NETBIRD_VERIFY.txt on VPS, 590_FORUM_SMTP_FAILED_CONFIG_DISABLED_PROOF.txt, 591_POST_INCIDENT_PUBLIC_STATUS_PROOF.txt, 592_POST_INCIDENT_FINAL_STABLE_STATE_PROOF.txt. + +## FORUM_SMTP_ONE_SHOT_TEST_OK_20260701 +- One controlled SMTP test from forum-prod to mail.pvepro.ru:587 succeeded using rotated credentials. +- No persistent forum SMTP config was enabled. +- One-shot secret/config files were removed after the test. +- Mailcow and NetBird remained reachable after the test. +- Proofs: /root/evidence/594_FORUM_SMTP_ONE_SHOT_TEST_PROOF.txt and /root/evidence/595_SMTP_TEST_AFTERCHECK_PROOF.txt. + +## SMTP_ONESHOT_OK_STABLE_FINAL_20260701 +- Rotated SMTP credential was tested once from forum-prod to mail.pvepro.ru:587 successfully. +- No persistent SMTP config or secret was left on forum-prod after the one-shot test. +- Mailcow and NetBird remained reachable after the test. +- Forum websites remained healthy. +- Persistent forum SMTP is still not enabled; enable it only with an explicit command and hidden password input. +- Proofs: 594_FORUM_SMTP_ONE_SHOT_TEST_PROOF.txt, 595_SMTP_TEST_AFTERCHECK_PROOF.txt, 596_SMTP_ONESHOT_OK_FINAL_STABLE_PROOF.txt. + +## FORUM_PERSISTENT_MSMTP_ENABLED_20260701 +- Persistent msmtp-mta transport enabled on forum-prod as /usr/sbin/sendmail. +- SMTP relay: mail.pvepro.ru:587 STARTTLS, sender noreply@pvepro.ru. +- Secret stored only as root:www-data 0640 base64 file under /etc/msmtp/pvepro.env; no raw SMTP_PASS file remains. +- PHP mail() as www-data succeeded after persistent config installation. +- Mailcow and NetBird remained reachable after enabling persistent forum SMTP. +- Proofs: /root/evidence/597_ENABLE_FORUM_PERSISTENT_MSMTP_PROOF.txt and /root/evidence/598_PERSISTENT_MSMTP_FINAL_VERIFY_PROOF.txt. + +## FORUM_PERSISTENT_SMTP_FINAL_STABLE_20260701 +- Persistent forum SMTP via msmtp is enabled on forum-prod. +- PHP mail() as www-data succeeded after installation. +- Forum domains remain healthy over HTTPS. +- Mailcow and NetBird remain reachable after enabling persistent forum SMTP. +- Secret storage is root:www-data 0640 under /etc/msmtp/pvepro.env; raw [SENSITIVE_PATH] SMTP_PASS file is absent. +- Snapshot after enable: forum-smtp-on-080840Z. +- Proofs: 597_ENABLE_FORUM_PERSISTENT_MSMTP_PROOF.txt, 598_PERSISTENT_MSMTP_FINAL_VERIFY_PROOF.txt, 600_FORUM_PERSISTENT_SMTP_FINAL_STABLE_PROOF.txt. + +## FORUM_PROD_FRESH5_MAILRU_CLOUD_BACKUP_OK_20260701 +- Existing Mail.ru Cloud rclone crypt remotes on pve02 were reused; no new cloud account was configured. +- Old codevipe-cloud-backup.timer was stale after fresh five-forum rebuild and is disabled. +- New pve02 timer forum-prod-fresh5-cloud-backup.timer uploads current forum-prod local archives from VM160 /var/backups/forums. +- Current backup contains all five forums: codevipe, gamevipe, hkmods, zakrutim, dsmods. +- Upload target: pve02-mail-01/02/03/04-crypt under app/forums/forum-prod/fresh5/. +- Archive is split into 1500M parts before upload, with SHA256SUMS and MANIFEST.txt. +- First fresh5 cloud upload proof: /root/evidence/608_FORUM_PROD_FRESH5_CLOUD_BACKUP_FIRST_RUN_PROOF.txt. + +## FORUM_PROD_FRESH5_MAILRU_CLOUD_RESTORE_DRILL_OK_20260701 +- Restore drill from Mail.ru Cloud crypt remote completed for forum-prod fresh5 backup. +- Remote source used: pve02-mail-01-crypt app/forums/forum-prod/fresh5 latest timestamp. +- Downloaded split parts, MANIFEST.txt, SHA256SUMS and SHA256SUMS.local were verified. +- Part SHA256 verification passed. +- Split archive was reconstructed and full archive hash matched SHA256SUMS.local. +- tar listing confirmed DB dumps and file archives for codevipe, gamevipe, hkmods, zakrutim and dsmods. +- Proof: /root/evidence/612_FORUM_FRESH5_CLOUD_RESTORE_DRILL_CORRECTED_PROOF.txt. + +## FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_OK_20260701 +- Five public XenForo forums are healthy over HTTPS. +- Mailcow admin and NetBird UI are reachable after persistent forum SMTP enablement. +- Persistent forum SMTP via msmtp is enabled and tested. +- Local backup exists on VM160 under /var/backups/forums. +- Mail.ru Cloud fresh5 backup is enabled on pve02 via forum-prod-fresh5-cloud-backup.timer. +- Old CodeVipe-only cloud timer is disabled. +- Cloud restore drill passed: split parts verified, archive reconstructed, SHA256 matched, tar contained DB/file archives for all five forums. +- Final rollup proof: /root/evidence/614_FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_PROOF.txt. + +## FORUM_XENFORO_JOB_RUNNER_CONFIGURED_20260701 +- XenForo CLI job runner configured inside VM160 forum-prod. +- Script: /root/scripts/forum-xenforo-run-jobs.sh. +- Cron: /etc/cron.d/forum-xenforo-run-jobs every 5 minutes. +- Purpose: process XenForo job queues and cron tasks independent of forum traffic. +- First run proof: /root/evidence/617_XENFORO_RUN_JOBS_ONCE_PROOF.txt. +- Cron verification proof: /root/evidence/618_XENFORO_JOB_RUNNER_CRON_CONFIG_PROOF.txt. + +## FORUM_XENFORO_CRON_DAEMON_AND_JOBRUNNER_OK_20260701 +- cron daemon inside VM160 forum-prod is active/enabled and was proven by temporary /etc/cron.d execution proof. +- XenForo job runner script executed successfully after cron daemon verification. +- Due XenForo cron count returned to zero after run. +- Proof: /root/evidence/621_FORUM_CRON_DAEMON_JOBRUNNER_VERIFY_PROOF.txt. + +## FORUM_DBTECH_TOR_TIMEOUT_CLASSIFIED_20260701 +- Each forum had one historical XenForo error_log row for cURL timeout to check.torproject.org/torbulkexitlist. +- Source addon: DBTech/Security DragonByte Security 5.0.0. +- dbtech_security_tornodes option is 0 on all five forums. +- Classified as non-blocking historical external network timeout/noise; not a forum runtime failure. +- Error rows were not deleted. +- Proof: /root/evidence/622_TOR_TIMEOUT_CLASSIFICATION_FINAL_PROOF.txt. + +## FORUM_XENFORO_BUILTIN_MAIL_DELIVERED_BUT_SPAM_20260701 +- Built-in XenForo outgoing email test from site "Моды для Hollow Knight" was delivered to aleisaev@yandex.ru. +- Sender was noreply@pvepro.ru. +- Yandex placed the message in Spam with warning that links/images were disabled. +- This proves forum SMTP transport works, but deliverability/reputation needs tuning. +- Custom proof 623 is invalid due /root path permission issue and is superseded by this user-observed built-in test. +- Next evidence needed: Yandex raw message headers, especially Authentication-Results and DKIM-Signature. +- Proof: /root/evidence/627_XENFORO_BUILTIN_MAIL_DELIVERED_SPAM_CLASSIFICATION_PROOF.txt. + +## FORUM_YANDEX_DELIVERABILITY_SPAM_WITH_SPF_DKIM_PASS_20260701 +- Built-in XenForo outgoing email test was delivered to aleisaev@yandex.ru but placed in Spam. +- User-provided Yandex headers showed SPF pass and DKIM pass for pvepro.ru. +- Return-Path, From and DKIM domain aligned on pvepro.ru. +- Yandex spam score observed: X-Yandex-Spam: 4. +- Classification: SMTP/Mailcow/XenForo transport is OK; remaining issue is deliverability/reputation/content filtering. +- DNS change is not required based on this header proof. +- Next actions: mark as Not spam in Yandex, add/check pvepro.ru in Yandex Postmaster, warm up sender reputation, optionally plan per-forum sender domains later. + +## FORUM_PROJECT_FINAL_ACCEPTANCE_OK_20260701 +- Five XenForo forums are published and healthy over HTTPS: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- VM160 forum-prod is the active production forum VM on pve02 at [PRIVATE_IP]. +- Runtime services are configured: nginx, MariaDB, PHP 8.3 FPM, qemu-agent, cron, msmtp/sendmail transport. +- NPMplus on edge terminates TLS and proxies all five forums to VM160. +- Let’s Encrypt certificates are active for all five domains. +- Forum SMTP transport works through noreply@pvepro.ru; Yandex headers proved SPF pass and DKIM pass. Spam placement is classified as reputation/content deliverability, not server failure. +- XenForo job runner cron is configured and cron daemon execution was proven. +- Local backup is configured on VM160 and Mail.ru Cloud fresh5/all-forums backup is configured on pve02. +- Cloud restore drill passed: split parts verified, archive SHA matched, tar contained DB/file/config archives for all five forums. +- DBTech Tor timeout rows are classified as historical external timeout noise and were not deleted. +- Final proof: /root/evidence/629_FORUM_PROJECT_FINAL_ACCEPTANCE_PROOF.txt. + +## FORUM_NEXT_CHAT_HANDOFF_FULL_STATE_20260701 + +### Start point +- Start shell: root@pve01. +- Canonical truth files: + - /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md + - /etc/pve/31_HOMELAB_REFERENCE.md +- Before any infra command, strictly check both files and relevant context blocks. +- Every infra command must print: + - ERROR_REGISTER_CHECK=OK + - REFERENCE_CHECK=OK + +### Production forum VM +- VMID: 160. +- Name: forum-prod. +- Node: pve02. +- IP: [PRIVATE_IP]. +- OS: Debian 12. +- Runtime: nginx, MariaDB 10.11, PHP 8.3 FPM, qemu-agent, cron, msmtp/sendmail transport. +- Final accepted snapshot: forum-final-accepted-091934Z. +- Mail final snapshot: forum-mail-ok-091815Z. +- Postlaunch snapshot: forum-postlaunch-ok-085501Z. + +### Public forums +- codevipe.ru -> Форум CodeVipe. +- gamevipe.ru -> Форум GameVipe. +- hkmods.ru -> Моды для Hollow Knight. +- zakrutim.ru -> Консервирование и закрутки. +- dsmods.ru -> Секреты и моды Doom. +- All five are public HTTPS 200 with valid TLS. +- Root/www DNS points through edge public IP 95.84.154.183. +- NPMplus on edge terminates TLS and proxies to http://[PRIVATE_IP]:80. + +### Mail +- Forum sender: noreply@pvepro.ru. +- Transport: XenForo sendmail -> msmtp -> mail.pvepro.ru. +- Built-in XenForo mail test reached Yandex. +- Yandex headers showed SPF pass and DKIM pass for pvepro.ru. +- Yandex placed the test in Spam with X-Yandex-Spam: 4. +- Classification: transport/authentication OK; remaining issue is sender reputation/content filtering. +- DNS change is not required from the captured header proof. +- Manual next action: click “Не спам!” in Yandex and add/check pvepro.ru in Yandex Postmaster. + +### Backups and restore +- Local backup configured inside VM160. +- Local archive path pattern: /var/backups/forums/forum-backup-*.tar.gz. +- pve02 Mail.ru Cloud fresh5/all-forums backup configured. +- New timer: forum-prod-fresh5-cloud-backup.timer active/enabled. +- Old codevipe-only timer disabled/inactive. +- Cloud remotes: + - pve02-mail-01-crypt + - pve02-mail-02-crypt + - pve02-mail-03-crypt + - pve02-mail-04-crypt +- Latest verified cloud stage during final acceptance: 20260701T083354Z. +- Restore drill passed: split parts verified, archive SHA matched, tar contained DB/file/config archives for all five forums. + +### XenForo jobs and cron +- cron daemon was missing, then installed and enabled. +- /etc/cron.d execution was proven by temporary cron proof. +- XenForo job runner configured: + - /root/scripts/forum-xenforo-run-jobs.sh + - /etc/cron.d/forum-xenforo-run-jobs +- Purpose: process XenForo jobs and cron tasks independent of visitor traffic. + +### DBTech/Tor timeout +- Each forum had one historical xf_error_log row for cURL timeout to check.torproject.org. +- Source addon: DBTech/Security DragonByte Security 5.0.0. +- dbtech_security_tornodes=0 on all five forums. +- Classified as non-blocking historical external timeout noise. +- Rows were not deleted. + +### Key final proofs +- /root/evidence/629_FORUM_PROJECT_FINAL_ACCEPTANCE_PROOF.txt +- /root/evidence/628_FORUM_MAIL_DELIVERABILITY_FINAL_SNAPSHOT_PROOF.txt +- /root/evidence/627_YANDEX_HEADER_DELIVERABILITY_CLASSIFICATION_PROOF.txt +- /root/evidence/622_TOR_TIMEOUT_CLASSIFICATION_FINAL_PROOF.txt +- /root/evidence/621_FORUM_CRON_DAEMON_JOBRUNNER_VERIFY_PROOF.txt +- /root/evidence/614_FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_PROOF.txt +- /root/evidence/613_FORUM_FRESH5_CLOUD_RESTORE_DRILL_FINALIZE_PROOF.txt +- /root/evidence/612_FORUM_FRESH5_CLOUD_RESTORE_DRILL_CORRECTED_PROOF.txt +- /root/evidence/609_FORUM_PROD_FRESH5_CLOUD_BACKUP_FINALIZE_PROOF.txt +- /root/evidence/608_FORUM_PROD_FRESH5_CLOUD_BACKUP_FIRST_RUN_PROOF.txt +- /root/evidence/600_FORUM_PERSISTENT_SMTP_FINAL_STABLE_PROOF.txt +- /root/evidence/570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt +- /root/evidence/560_FORUM_BACKUP_SCRIPT_SETUP_PROOF.txt + +## PARKED_DOMAINS_PLACEHOLDER_PUBLIC_OK_20260701 + +- Parked/placeholder public page is live for three unused domains: newfi.ru, hapusya.ru and kingofwolk.ru. +- Public DNS root and www hostnames already point to edge public IP 95.84.154.183. +- Local placeholder service on edge-vm: homelab-parked-domains-http.service, enabled/active, serves marker PARKED_PAGE_OK on http://127.0.0.1:18088. +- NPMplus manual managed route files: /data/nginx/proxy_host/998.conf for newfi.ru, 997.conf for hapusya.ru and 996.conf for kingofwolk.ru. +- Public HTTPS validation from pve01 after final route-only finalize returned HTTP 200 and PARKED_PAGE_OK for all six hostnames: root and www for all three domains. +- Certificates were issued by certbot DNS-01 using Cloudflare API hooks; HTTP-01 must not be retried for these parked domains without a new plan because earlier attempts hit NPMplus default redirect/include-order behavior. +- Cloudflare token value is not stored in the reference; token file path only: /opt/npmplus/secure/parked_cf_token, root-owned mode 600. +- Certificate names on edge-vm: parked-newfi.ru, parked-hapusya.ru and parked-kingofwolk.ru; observed expiry during setup: 2026-09-29. +- Certbot deploy hook installed: /etc/letsencrypt/renewal-hooks/deploy/parked-domains-npmplus-deploy.sh. +- Health file: /var/lib/homelab-health/parked-domains-public.txt. +- Final route backup: /opt/npmplus/manual-backups/parked-stage16-route-only-20260701T155843Z. +- Final proof: /root/evidence/645_PARKED_DOMAINS_STAGE16_ROUTE_ONLY_FINALIZE_PROOF.txt. + +## GRAM1_ROOT_WWW_PLACEHOLDER_PUBLIC_OK_20260701 + +- gram1.ru root and www.gram1.ru are routed to the existing parked placeholder page on edge-vm. +- Existing gram1.ru subdomain routes were not changed. +- NPMplus managed route file: /data/nginx/proxy_host/995.conf. +- Upstream placeholder: http://127.0.0.1:18088. +- Certificate name on edge-vm: parked-gram1.ru. +- Certificate issuance mode: certbot DNS-01 with dedicated gram1 Cloudflare token file /opt/npmplus/secure/gram1_cf_token. +- Token value is not stored in reference; token file is root-owned mode 600. +- Certbot deploy hook: /etc/letsencrypt/renewal-hooks/deploy/gram1-root-npmplus-deploy.sh. +- Health file: /var/lib/homelab-health/gram1-root-placeholder.txt. +- Closure proof: /root/evidence/653_GRAM1_ROOT_WWW_PLACEHOLDER_CLOSE_STAGE19_PROOF.txt. + +## PVEPRO_ROOT_WWW_EDGE_LANDING_PUBLIC_OK_20260701 + +- pvepro.ru and www.pvepro.ru root/www were moved from the Mailcow VPS default web surface to an edge landing page. +- mail.pvepro.ru and nb.pvepro.ru were intentionally not changed. +- Cloudflare A records for pvepro.ru and www.pvepro.ru point to edge public IP 95.84.154.183. +- NPMplus managed route file: /data/nginx/proxy_host/994.conf. +- Landing service on edge-vm: homelab-pvepro-landing-http.service on http://127.0.0.1:18089. +- Landing marker: PVEPRO_LANDING_OK. +- Certificate name on edge-vm: landing-pvepro.ru. +- Certificate issuance mode: certbot DNS-01 with dedicated pvepro Cloudflare token file /opt/npmplus/secure/pvepro_cf_token. +- Token value is not stored in reference; token file is root-owned mode 600. +- Certbot deploy hook: /etc/letsencrypt/renewal-hooks/deploy/pvepro-root-npmplus-deploy.sh. +- Health file: /var/lib/homelab-health/pvepro-root-landing.txt. +- Closure proof: /root/evidence/658_PVEPRO_STAGE23_FINAL_READONLY_CLOSE_PROOF.txt. + +## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701 + +- taftauto.ru is the dacha router domain. +- Current public A record observed during audit: 194.33.48.131. +- Router model family: Netcraze-like, same as Moscow router family by operator statement. +- Safe private management access is not available yet. +- Do not expose the router admin interface publicly for certificate automation. +- Certificate auto-renewal/deploy to the router is intentionally blocked until a private access path exists. +- Recommended future path: establish VPN/NetBird/WireGuard or reverse-proxy/private management endpoint first; then issue DNS-01 certificate on a trusted node and deploy cert/key to the router only over that private path. +- Closure status: documented blocker, not runtime outage. +- Read-only audit proof: /root/evidence/654_PVEPRO_TAFTAUTO_EXTERNAL_READONLY_AUDIT_PROOF.txt. + +## DOMAIN_PORTFOLIO_FINAL_STATUS_20260701 + +- Public parked placeholder domains closed: newfi.ru, hapusya.ru, kingofwolk.ru. +- gram1.ru root and www.gram1.ru closed on the same placeholder page; existing gram1.ru service subdomains were not changed. +- pvepro.ru root and www.pvepro.ru closed on a separate edge landing page; mail.pvepro.ru and nb.pvepro.ru remain on the external VPS/IPs and were validated after the change. +- Forum domains remain XenForo on forum-prod through edge NPMplus: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- taftauto.ru is documented as blocked for certificate autodeploy until private router management access exists. +- Forum renewal one-shot proof from Stage18 returned OK. +- Parked certbot dry-run passed for newfi.ru and hapusya.ru; kingofwolk.ru dry-run hit transient Let’s Encrypt service-busy/rateLimited after a valid active certificate and working public HTTPS were already confirmed. +- Current finalization proof: /root/evidence/661_DOMAIN_PORTFOLIO_FINAL_CLOSE_AND_TAFTAUTO_BLOCKED_PROOF.txt. + +## TAFTAUTO_WG_PRIVATE_MANAGEMENT_OK_20260702 +- Public SSH 194.33.48.131:22 closed. +- WireGuard management path OK: edge [PRIVATE_IP] -> dacha router [PRIVATE_IP]. +- Router HTTP over WG returns HTTP/1.1 200 OK / Ndm-Sysmode: router. +- Proof: /root/evidence/665_TAFTAUTO_WG_PRIVATE_MANAGEMENT_OK_20260702_PROOF.txt + +## TAFTAUTO_WG_PSK_ROTATED_AND_MANAGEMENT_OK_20260702 +- WireGuard PSK rotated after leaked self-test. +- Dacha active interface after reimport: Wireguard1. +- Public SSH remains closed. +- Private management OK: edge [PRIVATE_IP] -> dacha [PRIVATE_IP] via WG. +- Proof: /root/evidence/666_TAFTAUTO_WG_PSK_ROTATED_AND_MANAGEMENT_OK_20260702_PROOF.txt + +## TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702 +- Supersedes proof 666 because proof 666 showed PUBLIC_SSH_22_STILL_OPEN. +- WireGuard PSK rotated. +- Public SSH closed. +- Private WG management OK: edge [PRIVATE_IP] -> dacha [PRIVATE_IP]. +- Proof: /root/evidence/667_TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702_PROOF.txt + +## TAFTAUTO_CERT_DNS01_ISSUED_20260702 +- DNS-01 certificate issued on edge-vm for taftauto.ru and www.taftauto.ru. +- Cert name: router-taftauto.ru. +- Cert path: /etc/letsencrypt/live/router-taftauto.ru/fullchain.pem. +- Key path: /etc/letsencrypt/live/router-taftauto.ru/privkey.pem. +- Proof: /root/evidence/668_TAFTAUTO_CERT_DNS01_ISSUED_20260702_PROOF.txt + +## TAFTAUTO_PUBLIC_EDGE_WG_ROUTER_FINAL_OK_20260702 +- Supersedes failed/partial proof 669 because direct SSH was open during that run. +- taftauto.ru and www.taftauto.ru point to edge public IP 95.84.154.183. +- NPMplus route: /opt/npmplus/nginx/proxy_host/993.conf. +- TLS terminates on edge using certbot DNS-01 cert router-taftauto.ru. +- Upstream is private WireGuard path: edge [PRIVATE_IP] -> dacha router [PRIVATE_IP]:80. +- Public HTTPS returns router page with x-taftauto-router: managed and ndm-sysmode: router. +- Direct dacha public SSH is closed; WG SSH remains open. +- Proof: /root/evidence/670_TAFTAUTO_PUBLIC_EDGE_WG_ROUTER_FINAL_OK_20260702_PROOF.txt + +## HOMEPAGE_EXTERNAL_SMARTAPE_AND_SITES_OK_20260702 +- Homepage services.yaml repaired with YAML-aware edit after bad indentation issue. +- SmartApe card added to External Homelabs. +- Router Moscow card removed. +- Sites category added with 11 site cards: CodeVipe, GameVipe, HKMods, Zakrutim, DSMods, Newfi, Hapusya, KingOfWolk, Gram1, PVEPro, TaftAuto. +- Proof: /root/evidence/671_HOMEPAGE_EXTERNAL_SMARTAPE_AND_SITES_OK_20260702_PROOF.txt + +## TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK_20260702 +- Certbot renew dry-run for router-taftauto.ru succeeds with manual DNS-01 hooks. +- Deploy hook installed: /etc/letsencrypt/renewal-hooks/deploy/taftauto-router-npmplus-deploy.sh. +- Deploy hook copies renewed cert/key into NPMplus /data/tls/router-certs/taftauto.ru and reloads nginx after nginx -t. +- Proof: /root/evidence/672_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK_20260702_PROOF.txt + +## TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_INSTALLED_DRYRUN_RATELIMITED_20260702 +- Supersedes partial proof 672. +- Deploy hook installed and manual deploy invocation works. +- NPMplus cert copy and nginx config validate OK. +- Public taftauto.ru remains OK. +- Certbot dry-run retry is deferred due Let's Encrypt rateLimited / Service busy. +- Proof: /root/evidence/673_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_INSTALLED_DRYRUN_RATELIMITED_20260702_PROOF.txt + +## VPN_GRAM1_ALIAS_ROUTES_REMOVED_20260702 +- Removed obsolete NPMplus route aliases without Homepage cards: auth.vpn.gram1.ru, gitea.vpn.gram1.ru, homepage.vpn.gram1.ru, kuma.vpn.gram1.ru. +- NPMplus nginx config validates after removal. +- Proof: /root/evidence/674_VPN_GRAM1_ALIAS_ROUTES_REMOVED_20260702_PROOF.txt + +## HOMELAB_HOMEPAGE_AND_TAFTAUTO_FINAL_OK_20260702 +- Homepage final layout/category state confirmed: YAML validates, categories use row/columns layout, logs show no YAMLException. +- TaftAuto public HTTPS works for taftauto.ru and www.taftauto.ru through edge/NPMplus. +- Direct public SSH to dacha router is closed. +- Private WG SSH to dacha router remains open. +- NPMplus nginx config validates. +- Proof: /root/evidence/677_HOMELAB_HOMEPAGE_AND_TAFTAUTO_FINAL_OK_20260702_PROOF.txt + +## CLOUDFLARE_TOKEN_AUDIT_20260702 +- Cloudflare token files audited without printing secrets. +- Token files exist, expected permissions were checked, tokens verify active, and expected zone access was checked. +- Proof: /root/evidence/678_CLOUDFLARE_TOKEN_AUDIT_20260702_PROOF.txt + +## EVIDENCE_REVIEW_INDEX_20260702 +- Read-only evidence index created for review/superseded proof cleanup planning. +- No evidence files were deleted or modified. +- Proof: /root/evidence/679_EVIDENCE_REVIEW_INDEX_20260702_PROOF.txt + +## EVIDENCE_SUPERSEDED_MAP_20260702 +- Evidence superseded map created. No evidence files were deleted. +- 666, 669, 672_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK, and 676 are not authoritative for final state. +- 677 is current final Homepage/TaftAuto authority; 678 is current Cloudflare token audit authority. +- Proof: /root/evidence/680_EVIDENCE_SUPERSEDED_MAP_20260702_PROOF.txt + +## HOMEPAGE_LOCAL_HEALTH_MONITORS_OK_20260702 +- Local edge health endpoint is used for Homepage monitoring of Роутер Москва and NPMplus. +- Роутер Москва href remains external, while siteMonitor uses http://[PRIVATE_IP]:18091/router-moscow. +- NPMplus siteMonitor uses http://[PRIVATE_IP]:18091/npmplus. +- Homepage YAML validates and logs show no YAMLException. +- Proof: /root/evidence/681_HOMEPAGE_LOCAL_HEALTH_MONITORS_OK_20260702_PROOF.txt + +## HOMEPAGE_SELF_CARD_REMOVED_FROM_CORE_20260702 +- Removed Homepage self-referential card from Core. +- Homepage YAML validates and logs show no YAMLException. +- Proof: /root/evidence/682_HOMEPAGE_SELF_CARD_REMOVED_FROM_CORE_20260702_PROOF.txt + +## CERT_RENEW_HOOKS_READONLY_AUDIT_20260702 +- Read-only audit of cert renewal cron entries, renewal configs, deploy hooks, and related script presence completed. +- No certbot renew/dry-run was executed. +- Proof: /root/evidence/683_CERT_RENEW_HOOKS_READONLY_AUDIT_20260702_PROOF.txt + +## CERT_RENEW_MAPPING_READONLY_AUDIT_20260702 +- Read-only mapping audit completed for renewal confs, deploy hooks, and forum renewal script. +- No certbot renew/dry-run was executed and no secrets were printed. +- Proof: /root/evidence/684_CERT_RENEW_MAPPING_READONLY_AUDIT_20260702_PROOF.txt + +## CERT_DEPLOY_HOOK_LINEAGE_GUARDS_OK_20260702 +- Deploy hooks hardened with RENEWED_LINEAGE guards for gram1, parked domains, and pvepro. +- TaftAuto hook already had lineage guard and remains guarded. +- bash -n validates all checked deploy hooks. +- No certbot renew/dry-run was executed. +- Proof: /root/evidence/685_CERT_DEPLOY_HOOK_LINEAGE_GUARDS_OK_20260702_PROOF.txt + +## PUBLIC_HTTPS_AND_CERT_EXPIRY_READONLY_20260702 +- Read-only public HTTPS and certificate expiry snapshot completed for forum, parked, gram1, pvepro, and taftauto domains. +- No certbot renew/dry-run was executed. +- Proof: /root/evidence/686_PUBLIC_HTTPS_AND_CERT_EXPIRY_READONLY_20260702_PROOF.txt + +## NPMPLUS_ROUTE_CERT_MAPPING_READONLY_20260702 +- Read-only NPMplus route to certificate mapping audit completed. +- Expected proxy routes were found and expected certificate files are present and valid for more than 30 days. +- No certbot renew/dry-run was executed. +- Proof: /root/evidence/687_NPMPLUS_ROUTE_CERT_MAPPING_READONLY_20260702_PROOF.txt + +## TODAY_PROOFS_SECRET_SCAN_READONLY_20260702 +- Read-only filename-only secret pattern scan completed for today proof files and active reference/error register. +- No matching secret value patterns were found. +- Proof: /root/evidence/688_TODAY_PROOFS_SECRET_SCAN_READONLY_20260702_PROOF.txt + +## HOMELAB_20260702_SESSION_CLOSURE_OK +- Session closure proof created for final OK chain 677-688. +- TaftAuto, Homepage, Cloudflare tokens, evidence map, cert hooks, deploy guards, public HTTPS, route/cert mapping, and secret scan are documented. +- Proof: /root/evidence/689_HOMELAB_20260702_SESSION_CLOSURE_OK_PROOF.txt + +## CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702 +- edge-vm CrowdSec CAPI is registered and enabled through NetBird exit route. +- Client peer: edge-vm.netbird.selfhosted / 100.100.60.182. +- Primary exit routing peer: relay.netbird.selfhosted / 100.100.19.1 / Moldova. +- Backup egress peer present: mail.netbird.selfhosted / 100.100.147.204 / USA. +- NetBird exit route proof on edge-vm: wg allowed-ips includes 0.0.0.0/0 via relay; ip route get 1.1.1.1 uses wt0 table 7120. +- External trace after route: 85.121.4.192 / OTP, not home 95.84.154.183 / ARN. +- CrowdSec proof: cscli lapi status OK, cscli capi status OK, CAPI sharing/blocklist pull enabled, no DISABLE_ONLINE_API or -no-capi flags in active compose. +- WARP is intentionally absent and must not be reintroduced for this path. +- Unauthenticated https://api.crowdsec.net/v3/watchers/login returning HTTP 403 is expected network reachability proof. +- Final proof: /root/evidence/771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_20260702T191057Z_PROOF.txt + +## HOMEPAGE_RU_LAYOUT_MONITORING_PROPOSALS_20260702 +- Homepage live config is on core-apps VM [PRIVATE_IP] under /opt/stacks/homepage/config. +- Groups/descriptions were normalized to Russian. +- Layout is row/6-columns for all active groups. +- siteMonitor fields are preserved; monitors must be repaired, not removed. +- Cloudflare card must be left untouched by explicit operator request. +- Red badges after migration were primarily caused by Homepage container resolving *.vpn.gram1.ru to edge NetBird IP and timing out; fixed by mapping vpn hostnames to edge LAN [PRIVATE_IP] in Homepage compose extra_hosts. +- Router Moscow monitor uses edge health endpoint http://[PRIVATE_IP]:18091/router-moscow. +- NPMplus monitor uses edge health endpoint http://[PRIVATE_IP]:18091/npmplus. +- Missing-card proposal report: /var/lib/homelab-health/homepage-missing-card-proposals-final.txt. + +## UPTIME_KUMA_MISSING_MONITOR_AND_PROPOSALS_20260702 +- Uptime Kuma lives on monitoring VM [PRIVATE_IP]. +- Added one missing monitor: NPMplus Edge Health -> http://[PRIVATE_IP]:18091/npmplus. +- Uptime Kuma DB backup was created before DB mutation under /root/uptime-kuma-manual-backups/773a-* on monitoring VM. +- DB integrity after insertion was OK. +- Deep monitoring proposal report: /var/lib/homelab-health/uptime-kuma-monitoring-proposals.txt. +- Cloudflare must not be touched by operator request. +- Recommended next monitors are P0 only after operator approval: Router Moscow Health, Backup/Restore Dashboard, Restore Drill Index, NetBird Peers, CrowdSec CAPI, Paperless Restore. + +## UPTIME_KUMA_NPMPLUS_ADMIN_REPAIR_20260702 +- Existing Uptime Kuma monitor "NPMplus Admin" was red because it checked https://[PRIVATE_IP]:81/. +- NPMplus admin is intentionally localhost-bound on edge-vm, so monitoring VM should not check the admin UI directly. +- Fixed monitor target to edge health endpoint: http://[PRIVATE_IP]:18091/npmplus. +- Monitor was repaired, not deleted. +- Cloudflare was not touched. + +## UPTIME_KUMA_APPROVED_MONITOR_BATCH_20260702 +- Approved Uptime Kuma monitor batch installed after Homepage/Kuma analysis. +- Existing NPMplus Admin monitor was repaired to http://[PRIVATE_IP]:18091/npmplus, not deleted. +- Edge health wrapper /usr/local/sbin/router-moscow-health provides safe endpoints on [PRIVATE_IP]:18091 for router-moscow, npmplus, loki, alloy, cadvisor, registry-cache, socket-proxy, diun, kopia and crowdsec. +- Cloudflare was not touched by explicit operator request. +- Uptime Kuma DB backups are under /root/uptime-kuma-manual-backups/773d-approved-batch-* on monitoring VM before the DB mutation. + +## UPTIME_KUMA_PENDING_HEALTH_MONITORS_FIX_20260702 +- Four pending Uptime Kuma keyword monitors were repaired by using local pve01 health endpoint paths instead of public backup.gram1.ru paths: + - Backup Restore Dashboard -> http://[PRIVATE_IP]:9101/backup-restore-dashboard.txt + - Restore Drill Index -> http://[PRIVATE_IP]:9101/restore-drill-index.txt + - Paperless Restore -> http://[PRIVATE_IP]:9101/paperless-restore.txt + - AdGuard Rewrite Sync -> http://[PRIVATE_IP]:9101/adguard-rewrite-sync.txt +- Cloudflare was not touched. +- Monitors were repaired, not deleted. + +## UPTIME_KUMA_FOUR_HEALTH_SOURCES_REPAIRED_20260702 +- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR. +- Repaired pve01 health source files under /var/lib/homelab-health: + - backup-restore-dashboard.txt + - restore-drill-index.txt + - paperless-restore.txt + - adguard-rewrite-sync.txt +- Kuma monitors point to local pve01 health endpoints on http://[PRIVATE_IP]:9101/. +- Cloudflare was not touched. +- Monitors were repaired, not deleted. + +## DOCKGE_DISCONNECTED_HOSTS_AND_STALE_STACKS_20260702 +- Dockge is currently present only on edge-vm. +- core-apps and monitoring run Docker containers but have no Dockge/Dockge-agent detected. +- Dockge "2 not connected" should be interpreted as two disconnected Docker hosts unless later evidence shows otherwise. +- No stopped containers were found across edge-vm, core-apps and monitoring during analysis. +- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring. +- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification. + +## DOCKGE_REMOTE_AGENTS_INSTALLED_20260702 +- Dockge main instance runs on edge-vm [PRIVATE_IP]. +- Remote Dockge agents were installed on: + - core-apps [PRIVATE_IP]:5001 + - monitoring [PRIVATE_IP]:5001 +- Edge Dockge agent table stores the two agent URLs with username/password; password must never be printed. +- No runtime containers were deleted. +- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first. + +## DOCKGE_REMOTE_AGENTS_AND_STALE_ARCHIVE_FINAL_20260702 +- Dockge main instance: edge-vm [PRIVATE_IP]. +- Remote agents connected: core-apps [PRIVATE_IP]:5001, monitoring [PRIVATE_IP]:5001. +- Stale moved edge stack folders archived under /opt/dockge-stale-archive/20260702T230442Z on edge-vm; no runtime containers were deleted. +- External remote stacks exposed to Dockge using bind mounts: /opt/vaultwarden-compose, /opt/gotify-compose, /opt/uptime-kuma-compose into /opt/stacks. +- Final target: remote not_visible count must be 0 and edge stale remaining matches must be 0. + +## DOCKGE_FINAL_MANAGED_STACKS_20260702 +- Dockge main stack, npmplus and remote dockge-agent stacks were recreated from /opt/stacks so Dockge can manage them. +- Edge Dockge listens on 127.0.0.1:5001; LAN probe to [PRIVATE_IP]:5001 may return 000 and is not the correct health proof. +- Remote agents listen on [PRIVATE_IP]:5001 and [PRIVATE_IP]:5001. +- Stale moved stacks were archived, not deleted. + +## SERVICE_SETUP_ROADMAP_DEEP_20260702 +- Configuration order: secrets, DNS/proxy/TLS, backup/restore, monitoring, SSO, docs, core data apps, sensitive apps, productivity, media, automation, utilities. +- Do not configure data-heavy or automation services before backup and monitoring are proven. +- Per-service done criteria: URL, TLS, admin, SSO decision, volumes, backup, restore test, Kuma, Homepage, docs. + +## PHASE00_FREEZE_CLOSED_20260702 +- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain. +- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml. +- Old systemd unit homelab-registry-cache.service was disabled and masked because it recreated the container without Compose labels. +- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/. +- Next phase: Vaultwarden and break-glass secret baseline. + +## PHASE00_FREEZE_CLOSED_20260702 +- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain. +- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml. +- Old systemd unit homelab-registry-cache.service was archived and masked because it recreated the container without Compose labels. +- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/. +- Next phase: Vaultwarden and break-glass secret baseline. + +## PHASE00_FREEZE_CLOSED_20260702 +- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain. +- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml. +- Old systemd unit homelab-registry-cache.service was archived and masked with /dev/null because it recreated the container without Compose labels. +- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/. +- Next phase: Vaultwarden and break-glass secret baseline. + +## OPERATOR_RULE_CLOSE_TAILS_IMMEDIATELY_20260714 +- Жёсткое правило: хвосты не оставлять. Любая проблема, временный артефакт, неудачная проверка, блокер или анти-регрессия, обнаруженные в текущем scope, должны быть устранены и закрыты до перехода к следующей задаче. +- После любой неуспешной команды текущая задача остаётся активной до установления точной причины, исправления, проверки исправления, добавления анти-регрессии, очистки временных артефактов и выпуска закрывающего proof. +- Этап разрешено помечать CLOSED только при UNRESOLVED_TAIL_COUNT=0, BLOCKER_COUNT=0, TEMPORARY_ARTIFACT_COUNT=0, проверенном rollback, зелёном health, обновлённых proof и reference. +- Запрещено откладывать выявленный хвост только ради удобства или перехода к следующему этапу. +- Если технически необходим отдельный подэтап, он должен быть ограничен точным scope, выполнен и закрыт немедленно до возврата к основной работе. +- Исключения: внешняя зависимость, опасная неоднозначность, риск раскрытия секрета, отсутствующий обязательный файл или явное решение пользователя. В таком случае статус должен быть BLOCKED или OPEN с точным блокером; статус CLOSED запрещён. +- Следующий этап не начинается, пока текущий хвост не закрыт. +- Proof: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/report.txt +- Secret scan: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/strict-secret-scan.txt + +## CLUSTER_ADMIN_STAGE4H_SCOPE_DEFINED_20260714 +- Stage4G остаётся закрытым и стабильно работает в режиме observe-notify. +- Закрыты два хвоста dependency-аудита: глобальный LIMIT всего UNION ALL и узкий шаблон collector_patch_required. +- Migration plan содержит collector_patch_required=true; dynamic contract содержит collector_patch_required_after_stage4c=true. Это два подтверждения одного обязательного требования. +- Migration 003 не применена, 12 новых столбцов отсутствуют. +- Migration выполняет полный UPDATE observations, четыре SET NOT NULL, пять VALIDATE CONSTRAINT и пять неконкурентных CREATE INDEX. +- Текущий collector не формирует обязательный provenance-набор; готовый collector patch отсутствует. +- Production adapters и production apply phase отсутствуют. +- Отдельный apply migration 003 без collector patch запрещён. +- Безопасный порядок: collector patch candidate → временная БД и acceptance/rollback → контролируемый migration+collector cutover → adapter integration. +- Текущий этап открыт: 4H_COLLECTOR_PROVENANCE_PATCH_AND_MIGRATION003_PREAPPLY_READINESS. +- Предыдущие хвосты закрыты; Stage4H не считается CLOSED до реализации и полного seal. +- Proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt +- Secret scan: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/strict-secret-scan.txt + +## CLUSTER_ADMIN_STAGE4H_COLLECTOR_BASELINE_AUDIT_CLOSED_20260714 +- Исправленный baseline-аудит collector завершён. +- collector.py: 113 строк, SHA256 подтверждён. +- Привилегированные файлы нельзя читать через caller-side `< file` перед sudo; применён sudo wc без внешнего перенаправления. +- Фактические поля collector_runs: finished_at и error_text; completed_at и error отсутствуют. +- Найдены один canonical source instance и один collector_runs_foundation_enrich_trg. +- Trigger обогащает collector_runs полями source_instance_id и run_key. +- Текущий collector ещё не пишет provenance-поля observations; migration 003 не применена. +- Production осталась 0|0|OK, timer активен, failed units отсутствуют. +- Baseline-аудит закрыт без хвостов; Stage4H остаётся OPEN для isolated collector provenance patch candidate. +- Proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt +- Secret scan: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/strict-secret-scan.txt + +## CLUSTER_ADMIN_STAGE4H_COLLECTOR_PATCH_CANDIDATE_CLOSED_20260714 +- Из точного live collector создан isolated provenance patch candidate; production collector не изменён. +- Candidate пишет все 12 migration003 provenance-полей и применяет SHA256 fallback-дедупликацию. +- Metadata разрешает только instance, job, mountpoint и device, ограничена по типам и размеру. +- Candidate скомпилирован локально и проверен на VM180 Python 3.11.2 от clusteradmin. +- SSH stdin harness исправлен: bash -s получает ровно path, SHA256 и bytes; первый аргумент дополнительно проверяется по безопасному шаблону. +- Self-test и отрицательный CLI-тест RC64 прошли; временный remote-файл удалён. +- Production осталась 0|0|OK; live collector и timer не изменены. +- Неудачная попытка закрыта без хвостов; Stage4H остаётся OPEN до temporary-DB acceptance и controlled cutover. +- Candidate root: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z +- Proof: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/report.txt +- Secret scan: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/strict-secret-scan.txt + +## OPERATOR_RULE_NO_OVERSIZED_MONOLITH_COMMANDS_20260714 +- Запрещены чрезмерно длинные интерактивные однострочные команды с глубокой вложенностью SSH, SQL, Python, awk и кавычек. +- Рекомендуемый предел интерактивной команды: 8000 байт. Превышение допускается только для простого текста без вложенных языков. +- Сложная операция оформляется как отдельный versioned task-скрипт с contract, syntax-check, dry-run, manifest, rollback и proof. +- Создание task-скрипта и его запуск выполняются разными короткими командами. +- Remote stdout/stderr всегда сохраняются до проверки RC; запрещено терять вывод из-за errexit-sensitive command substitution. +- Перед запуском проверяются SHA256, размер, владелец, режим и синтаксис task-скрипта. +- Обрезанная, повреждённая или частично вставленная команда считается невыполненной; сначала проводится side-effect audit. +- Хвост parse-ошибки закрывается только после доказательства отсутствия изменений и временных артефактов. +- Proof: /var/lib/homelab-change-sets/operator-command-size-guard-v1/20260714T182653Z/report.txt + +## HOMELAB_ADMIN_CLI_CONTRACT_20260714 +- Поддерживаемые команды: version, status, lint TASK_DIR, validate TASK_DIR, run TASK_DIR PHASE. +- Отдельная команда --help отсутствует. Неизвестный аргумент печатает usage и штатно возвращает RC=64. +- Нельзя трактовать RC=64 от --help как неисправность runner. +- Для проверки доступности runner использовать homelab-admin version; для структуры CLI использовать зафиксированную usage-строку. +- Proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v1/20260714T183338Z/report.txt + +AUTOMATIC_CLUSTER_CONTEXT_REFRESH_20260721 +Collector: /usr/local/sbin/homelab-context-collect +Scheduler: hourly at minute 17 via /etc/cron.d/homelab-context-refresh +Destination: immutable Gitea runtime history with CONTEXT-AUTO UTC command IDs. + +AUTOMATIC_CLUSTER_CONTEXT_REFRESH_DIRECT_20260721 +Publisher: /usr/local/sbin/homelab-context-refresh-direct +Schedule: hourly at minute 17. +Destination: immutable assistant-runtime history with CONTEXT-AUTO UTC IDs. +DETAIL=SECTION_REFERENCE_REGISTER_END +DETAIL=SECTION_ERROR_REGISTER_BEGIN +# HOMELAB ASSISTANT ERROR REGISTER + +Назначение: перед каждой следующей командой сверяться с этим файлом. + +## Критические ошибки ассистента +1. Повторно дал слишком большой интерактивный paste в shell. +2. Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт. +3. Дал генератор справочника прямо в терминал вместо безопасного маленького шага. +4. Нарушил своё же правило: не давать длинные вложенные команды с кавычками. + +## Жёсткие правила перед каждой командой +CHECK-1: команда не должна быть большим paste. +CHECK-2: команда не должна содержать большой here-doc. +CHECK-3: команда не должна смешивать Markdown, backticks и shell-логику. +CHECK-4: команда не должна иметь вложенный ssh с несколькими уровнями кавычек. +CHECK-5: команда не должна печатать секреты. +CHECK-6: если создаётся файл, сначала маленький безопасный шаг, потом проверка. +CHECK-7: если команда длиннее 8 строк, её нельзя давать в интерактивный shell. +CHECK-8: для edge-vm использовать debian@[PRIVATE_IP] и sudo. +CHECK-9: для forum-prod использовать pve02 и ключ [SENSITIVE_PATH] +CHECK-10: Corosync не трогать без отдельного плана и rollback. + +## Текущие важные факты +Internal network: [PRIVATE_IP]/24. +Migration config: migration: secure,network=[PRIVATE_IP]/24. +Corosync remains on [PRIVATE_IP]/12/13. +VM160 forum-prod is not in Proxmox nightly backup. +VM130 edge-vm has secondary disk backup=0 risk. +05_edge_compose_safe.tgz quarantined. + +## Правило для справочника +Не генерировать большой справочник через интерактивную вставку. +Следующий справочник делать только маленькими append-блоками или через файл, созданный вне интерактивного shell. + +11. Ошибка: считать offhost OK после failed rsync. +Если rsync/scp упал, нельзя проверять latest offhost-файл без сверки имени. +Проверка должна подтвердить именно новый архив, например дату 2026_06_30-00_20_59. +Старый OFFHOST_ZSTD_OK не закрывает новый backup. + +12. Ошибка: широкий secret-поиск по /opt/stacks дал шум. +Не искать Cloudflare/SMTP сразу по /opt/stacks, MkDocs site, backups и HTML-отчётам. +Для ротации сначала смотреть live-конфиги: /opt/npmplus/tls/certbot/renewal/*.conf и реальные credentials-файлы из них. +Значения секретов не печатать; выводить только пути, ключи и redacted-поля. + +13. Ошибка: SQL с одинарными кавычками внутри одинарной SSH-команды ломается. +Нельзя писать sqlite SQL вида j.type in ('object','array') внутри ssh '...'. +Для sqlite через ssh использовать SQL без одинарных кавычек: char(36), length(j.atom), двойные внешние кавычки, либо отдельный файл. +Перед запуском проверять, что команда не содержит конфликтующих уровней кавычек. + +14. Ошибка: путать контекст входа и узел выполнения. +Стартовая точка оператора: root@pve01 / [PRIVATE_IP]. +edge-vm: ssh debian@[PRIVATE_IP], внутри использовать sudo. +pve02/pve03: ssh root@pve02 или ssh root@pve03. +forum-prod: заходить через pve02, ключ [SENSITIVE_PATH] +Перед каждой командой явно понимать: где выполняется локальная часть, где remote-часть, каким пользователем. + +15. Ошибка: повторно нарушено правило №13 после его добавления. +Снова был использован SQL JSON-path в одинарных кавычках внутри ssh '...'. +Запрещено использовать в remote sqlite SQL фрагменты вида '$.key' или '.key'. +Для JSON-path использовать только char(...), без одинарных кавычек внутри SQL. +Команду с ошибкой char(36)||.dns_provider считать битой и не использовать. + +16. Основополагающее правило: перед каждой командой ассистент обязан явно написать: Сверка с файлом ошибок: CHECK пройден. +После этой строки можно давать только одну короткую команду или один логический блок команд. +Нельзя выдавать команды без предварительной сверки с этим файлом ошибок. +Нельзя продолжать после собственной ошибки без записи ошибки в этот файл. + +19. Уточнение формата: не писать отдельную строку сверки перед каждой командой в пакете. +Сверка должна быть внутри самой команды первым логическим блоком: grep правил из файла ошибок и RULE_CHECK_OK. +Разрешено давать до трёх логически связанных команд сразу, по возможности в одну строку каждая. +Не писать фразу "ждём" после команд; указывать только контрольные строки результата. + +20. Ошибка: широкий XenForo SMTP-поиск по src дал шум исходников. +Для XenForo SMTP не grep-ать весь /src; сначала смотреть live src/config.php без значений, затем БД XenForo и xf_option/emailTransport. +Значения DB-паролей и SMTP-паролей не печатать. + +21. Ошибка: nested PHP php -r дал Parse error на forum-prod. +Команды 102-104 считать битыми: они не подтвердили и не изменили smtpSsl. +Не продолжать длинные вложенные ssh+php -r с кавычками; для forum-prod лучше войти в VM или использовать короткие mysql/php команды без вложенного кода. +Рабочими считать: DB backup перед SMTP, SMTP password update, JSON postcheck; SMTP auth ещё не закрыт из-за CODE=530 на AUTH LOGIN. + +22. Ошибка: самодельный base64 PHP для SMTP auth сломан. +Команда 108 дала PHP Parse error на smtpHost и пустой proof-файл. +Не использовать больше сгенерированные большие base64 PHP для SMTP/XenForo. +Для XenForo SMTP использовать короткие mysql-запросы к xf_option и готовые штатные механизмы XenForo. + +23. Ошибка: MariaDB не поддержала cast('true' as json) в JSON_SET. +Команда 111 не изменила smtpSsl: OLD_SMTPSSL=false, SQL ERROR 1064, NEW_SMTPSSL=false. +Перед боевым UPDATE сначала проверять JSON_SET синтаксис на тестовом JSON через SELECT. +Не считать PHP_FPM_RELOAD_OK подтверждением изменения БД. + +24. Ошибка: exit 1 в interactive-check закрыл SSH-сессию. +Нельзя использовать { echo RULE_CHECK_FAIL; exit 1; } в командах, выполняемых прямо в интерактивном root@pve01 shell. +При failed-check использовать безопасный шаблон: if grep ...; then echo RULE_CHECK_OK; ...; else echo RULE_CHECK_FAIL; fi. +Не делать fragile grep по точной строке с кавычками, если в файле ошибок строка содержит shell/SQL quoting. + +25. Ошибка анализа: не надо слепо включать smtpSsl=true при smtpPort=587. +smtpPort=587 обычно означает STARTTLS, а не implicit SMTPS. +CODE=530 на AUTH LOGIN означает, что ручной checker не сделал STARTTLS; это не доказывает неверный SMTP-пароль. +Сначала проверять AUTH через openssl s_client -starttls smtp, без вывода секрета. + +26. Ошибка: openssl -crlf вместе с ручным CRLF ломает SMTP AUTH checker. +Команда 116 получила 535 Invalid base64 data in continued response после 334 Username. +Это указывает на битую base64-строку checker-а, а не на неверный SMTP-пароль. +Не использовать -crlf, если команды уже отправляются с явным \r\n. +Для 587 оставлять smtpSsl=false и проверять STARTTLS корректным checker-ом. + +27. Ошибка: Python heredoc внутри ssh сломал not_ok диагностику. +Команда 138 пересобрала dashboard, но диагностический Python-блок упал с NameError из-за кавычек. +Не использовать heredoc Python внутри вложенного ssh; для dashboard verification использовать cat/grep JSON или простые команды. + +28. Ошибка: docker compose ls --format json | wc -l дал ложный COMPOSE_PROJECT_COUNT=1. +Причина: JSON-вывод может быть одной строкой/структурой, wc -l не является счётчиком проектов. +Для Docker Compose count использовать обычный docker compose ls --all + awk по строкам таблицы или JSON parser, не wc -l. + +29. Ошибка: monitoring compact status искал неверные имена health-файлов. +Факт: npmplus-cert-expiry.txt не является подробным файлом; подробный файл называется npmplus-certificate-expiry.txt, а статусный alias npmplus-cert-expiry-health.txt. +Факт: health-metrics.txt не обязан существовать как health-файл; health metrics могут публиковаться через node-exporter textfile collector. +Для Prometheus targets проверять HTTP_CODE/RC явно, не оставлять пустой блок PROMETHEUS_TARGETS_LOCAL. + +30. Ошибка/инцидент: Prometheus API HTTP_CODE=000 после monitoring reference. +Факт: в monitoring compact status curl к 127.0.0.1:9090 вернул HTTP_CODE=000, а raw inventory не показал контейнер prometheus среди monitoring containers. +Нельзя считать monitoring слой runtime-closed только по secret scan/reference proof; нужно отдельно проверить Prometheus container/API и обновить справочник по факту. + +31. Строгое правило: перед каждой командой сверяться с файлом ошибок и справочником. +Перед любым действием команда обязана проверять /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md и текущий 31_HOMELAB_REFERENCE.md. +Команда обязана печатать ERROR_REGISTER_CHECK=OK и REFERENCE_CHECK=OK до основного действия. +Если сверка не пройдена, команда не должна выполнять основное действие. Не использовать exit 1, чтобы не закрывать SSH-сессию. + +32. Ошибка: 153_PROMETHEUS_TARGETS_SETTLED_PROOF содержит SyntaxError из-за вложенного Python внутри SSH. +Факт: Python -c внутри ssh потерял кавычки вокруг /tmp/prom-targets-settled.json, data, activeTargets, labels, job, health. +Файл 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt нельзя считать валидным settled proof. +Для таких проверок не использовать вложенный Python; использовать curl + grep/sed/awk либо отдельный локальный файл-скрипт. + +34. Ошибка операционного ввода: полный терминальный лог MobaXterm попал в shell. +Факт: bash пытался выполнить строки Authenticating, MobaXterm banner, RULE_CHECK_OK, PROOF_OK и таблицы вывода как команды. +Такие выводы не являются инфраструктурным состоянием; считать невалидными только соответствующие failed command attempts. +Дальше давать короткие команды и не вставлять обратно полный transcript в shell. + +33. Security finding: root authorized_keys на PVE-нодах имел права 777. +Факт: inventory 158 показал [SENSITIVE_PATH] и backup-файлы с режимом 777 на pve01/pve02/pve03. +Нужно исправлять на [SENSITIVE_PATH]=700 и authorized_keys/authorized_keys.bak*=600, затем фиксировать before/after proof. + +35. Ошибка проверки: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED не доказал исправление прав. +Факт: AFTER всё ещё показал 777 для [SENSITIVE_PATH] на pve01/pve02/pve03. +Возможная причина: authorized_keys является symlink, а stat без -L показывает режим ссылки 777. +Нужно проверять stat -L целевого файла и считать 160 недостаточным proof. + +36. Quality check: Storage block needs integrity and pve03 capacity coverage review. +Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL. +Факт: pve03 /mnt/staging observed at 77%, close to WARN threshold 80%, while disk-space.txt did not visibly list pve03_staging. +Before closing storage layer, verify block integrity and pve03 capacity coverage. + +37. Coverage gap: pve03_staging missing from disk-space health coverage. +Факт: 169 showed pve03 /mnt/staging at 77%, close to WARN 80%, while disk-space.txt listed pve01_staging, pve02_staging and edge_immich_media only. +Before closing storage fully, add or document pve03_staging capacity monitoring. + +38. Quality check: Service Dependency Map block needs integrity review. +Факт: terminal output around command 222 showed paste artifact near "FAIL; fi". +Before closing Service Dependency Map, verify 178_REFERENCE_SERVICE_DEPENDENCY_MAP_BLOCK.txt and the reference section for shell/log garbage or truncation. + +39. Ошибка проверки: post deep audit BAD_MARKER scan caught prose false positive. +Факт: строка справочника с фразой No unresolved final REVIEW_REQUIRED... SyntaxError... была принята за BAD_MARKER. +Это не runtime/log мусор, а документированная фраза deep audit closure. В scan нужно исключать эту prose-строку. + +40. Ошибка/вывод: VM150 Mail cloud single-file upload failed with 413. +Факт: Mail/WebDAV rejected one 6.6G vzdump file with 413 Request Entity Too Large. +Решение: use VM150 recurring chunked script with 512M parts and download verification. + +41. Ошибка: post-backup-pass-slo-reconciliation checked pve01 router health on edge-vm. +Факт: first reconciliation wrote STATUS=ERROR because router-running-config-mail-cloud.txt is on pve01, not edge-vm. +Fix: build reconciliation from pve01 and edge-vm health files by correct owner. + +## ASSISTANT_COMMAND_BATCHING_RULE_20260630 +- Rule: combine related infrastructure checks, proof generation, reference updates and validation scans into one safe batch command when practical. +- Do not split work into many small commands when the operations are low-risk, read-only, or naturally part of one evidence block. +- Still keep mandatory preflight checks, short bounded commands, no large paste, no here-docs, no interactive shell traps, and no sensitive values in output. +- Split into smaller commands only when safety, rollback, quoting risk, or troubleshooting clarity requires it. + +## HOME_PORTAL_BASE64_APPLY_FAILURE_20260630 +- Prior home-portal apply attempt 312 failed because a large base64 Python payload decoded as invalid UTF-8. +- Rule: avoid large opaque base64 script payloads for homelab changes; use transparent bounded shell/perl/python commands instead. + +## HOMELAB_COMMAND_SAFETY_HARDENING_20260630 +- Rule: do not use large opaque payloads, generated base64 scripts, or hidden multi-line script blobs for homelab changes. +- Use transparent bounded shell/perl/python commands, or create a temporary readable script and validate it before execution. +- Any script used for changes must pass a syntax/parse check before it is allowed to modify files or restart services. +- Any Traceback, UnicodeDecodeError, SyntaxError, Permission denied, command-not-found, nonzero critical RC, or failed write marker must force REVIEW or FAILED, never OK. +- Success requires both runtime health and content-specific post-checks: expected files changed, expected cards exist, duplicates absent, service/container alive, and no new critical log pattern. +- HTTP 200 alone is not a success condition for configuration changes. +- If an apply step fails, the next command must explicitly record the failed attempt and then run a corrected transparent apply command. + +## HOME_PORTAL_API_ERROR_CLOSURE_RULE_20260630 +- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200. +- If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed. +- External informational widgets such as weather/Open-Meteo must not block the service launcher portal. + +## HOME_PORTAL_COMPLEX_QUOTING_FAILURE_20260630 +- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('. +- Cause: command was too complex and fragile due to nested shell/perl/python quoting. +- Rule: do not use deeply nested one-liners for config edits; first run short state checks, then apply small transparent edits. + +## HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RULE_20260630 +- Rule: Homepage cleanup validation must check active config files only, not logs or backup files. +- Active files are services.yaml, bookmarks.yaml, widgets.yaml, settings.yaml and docker.yaml under /opt/stacks/homepage/config. +- Matches in /logs/ or *.bak* are historical evidence only and must not make cleanup REVIEW unless the active config still contains the target. + +## HOME_PORTAL_LINK_VALIDATION_NPMPLUS_DEFAULT_RULE_20260630 +- Rule: Homepage link validation must not treat HTTP 200 as success if the body is the NPMPlus default “Congratulations” / host-not-set-up page. +- Green siteMonitor dots can be false positives when NPMPlus returns its default vhost page. +- A portal card is openable only when DNS, reverse-proxy host mapping, and target content are all valid. + +## ROUTER_CLI_STDIN_APPLY_FAILURE_20260630 +- Context: attempted Netcraze router ACL apply through SSH stdin/multiline for Homepage Moscow Router monitor fix. +- Evidence: 391_MOSCOW_ROUTER_EDGE_VM_WEBADMIN_ACL_APPLY_PROOF.txt. +- Failure: STDIN_SHOW_OK=no, APPLY_SKIPPED=stdin_show_failed, CONFIG_HAS_EDGE_5080=no, CONFIG_HAS_EDGE_5083=no. +- Rule: do not apply Netcraze config through stdin/multiline SSH; prove CLI input mode first. + +## ROUTER_CLI_PROBE_STDIN_CONSUMPTION_ERROR_20260630 +- Context: ACL syntax read-only probe loop executed only one command because ssh consumed the loop stdin. +- Evidence: 393_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_PROOF.txt showed COMMAND_COUNT=1. +- Fix pattern: use ssh -n or redirect SSH stdin away from the command-list loop for all future SSH-in-loop probes. + +## ROUTERBACKUP_READONLY_CLI_PERMISSION_LIMIT_20260630 +- Context: routerbackup user can run show running-config but cannot execute configuration commands such as access-list. +- Evidence: 394_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_FIXED_PROOF.txt. +- Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli]. +- Rule: do not attempt ACL/config changes as routerbackup; use admin/operator router access or router Web UI for write changes. + +## PROOF_SUMMARY_EXTRACTION_BLANK_20260630 +- Context: proof 395 tried to read COMMAND_COUNT/ZERO_RC_COUNT from proof files 393/394, but those summary fields were printed to console and were not persisted inside the proof files. +- Evidence: 395_ROUTER_CLI_ERROR_REGISTER_UPDATE_PROOF.txt has blank P393_COMMAND_COUNT/P394_COMMAND_COUNT/P394_ZERO_RC_COUNT fields. +- Rule: when creating follow-up proof summaries, derive counts from persisted proof body lines such as CMD_INDEX and CMD_RC, or write summary fields into the proof file itself. + +## ROUTER_ACL_UI_RULE_DELETE_AND_RESTORE_20260701 +- Context: while moving Moscow Router monitor rules to _WEBADMIN_Bridge1, two _WEBADMIN_Bridge0 Nextcloud Talk TURN rules were accidentally removed. +- Deleted/restored rules: Home [PRIVATE_IP]/24 to Nextcloud [PRIVATE_IP] TCP/UDP 3478. +- Verification proof: 399_ROUTER_ACL_RESTORE_AFTER_UI_PROOF.txt. +- Rule: after manual router Web UI ACL edits, compare _WEBADMIN_Bridge0, _WEBADMIN_Bridge1 and _WEBADMIN_GigabitEthernet1 before continuing. + +## ROUTER_MONITOR_NODE_LOOKUP_PROBE_BUG_20260701 +- Context: proof 400 tested Moscow router HTTPS monitor from Homepage container with custom Node lookup callback. +- Failure: NODE_HTTPS_RESOLVE_ERROR=ERR_INVALID_IP_ADDRESS and NODE_HTTPS_RESOLVE_INSECURE_ERROR=ERR_INVALID_IP_ADDRESS. +- Impact: Node HTTPS resolve subtest is invalid; curl --resolve results remain valid. +- Rule: do not rely on custom Node lookup callback until separately tested; prefer curl --resolve or simple direct Node HTTP checks. + +## ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701 +- Context: Moscow Router Homepage monitor after ACL fix. +- Evidence: proofs 399, 400, 401. +- Observed: edge-vm TCP to [PRIVATE_IP]:5080/5083 is OK, but router HTTP/HTTPS returns 403 for all tested paths. +- Impact: Homepage siteMonitor cannot be made green via direct router HTTP URL until router web service allows the Bridge1/Proxmox/protected segment or an explicit safe monitor endpoint is used. +- Rule: do not keep changing ACL for this symptom; ACL is already correct. Fix service access scope or use a dedicated monitor endpoint. + +## PY_COMPILE_PYC_PERMISSION_ERROR_20260701 +- Context: installing edge-vm Moscow router health endpoint. +- Mistake: assistant used `python3 -m py_compile` on a script installed under /usr/local/sbin as root. +- Actual impact: non-root syntax check attempted to create /usr/local/sbin/__pycache__ and failed with Permission denied. +- Recovery: validate syntax with `ast.parse` instead of py_compile, because it does not write .pyc files. +- Rule: do not use py_compile against root-owned system paths from an unprivileged user. + +## ROUTER_HOME_IP_DOCKER_ROUTE_CONFLICT_20260701 +- Context: Moscow Router Homepage siteMonitor attempt using http://[PRIVATE_IP]:5080. +- Evidence: proof 405 showed route to [PRIVATE_IP] via Docker bridge and Homepage container ECONNRESET. +- Impact: do not use [PRIVATE_IP] directly as Homepage siteMonitor from edge-vm. +- Mitigation: use dedicated edge-vm health endpoint that checks router TCP [PRIVATE_IP]:5080 and returns HTTP 200/503 for Homepage. + +## HOMEPAGE_ROUTER_MOSCOW_YAML_TITLE_SHAPE_MISMATCH_20260701 +- Context: applying Moscow Router Homepage siteMonitor health endpoint. +- Mistake: assistant apply script matched only property-style cards with `title`, but active Homepage YAML can use service-name-key style like `- Service Name:`. +- Evidence: proof 407 returned YAML_CHANGE_COUNT=0 and APPLY_SKIPPED=unexpected_match_count while health endpoint proof 406 was OK. +- Recovery: use YAML-aware updater that handles both property-style and service-name-key style cards. + +## HOMEPAGE_ROUTER_MOSCOW_APPLY_SCRIPT_SYNTAX_ERROR_20260701 +- Context: applying Moscow Router Homepage health endpoint. +- Mistake: assistant generated Python script with invalid f-string escaping. +- Evidence: proof 408 showed SyntaxError in /tmp/homepage-router-moscow-apply-fixed.py. +- Actual impact: YAML was not changed, so Homepage green dot could not appear. +- Recovery: stop applying blind updater; first discover exact active YAML shape, then apply minimal YAML-aware edit. +- Rule: validate generated Python syntax locally before scp/remote execution; avoid f-strings in generated maintenance scripts. + +## FORUM_PROD_VM160_FIRST_BOOT_SSH255_20260701 +- Context: clean rebuilt VM160 first boot. +- Symptom: PING_OK=yes and TCP22_OK=yes, but SSH_RC=255. +- Rule: do not proceed with OS baseline until SSH failure is diagnosed; likely old known_hosts key or cloud-init/root-key issue. + +## FORUM_PROD_VM160_NESTED_HOSTNAME_PROOF_QUOTING_20260701 +- Context: VM160 first SSH proof after rebuild. +- Issue: command substitution $(hostname) inside nested ssh was expanded on pve02 before entering VM160. +- Impact: VM_HOSTNAME=pve02 in proof 426 is not valid VM identity evidence. +- Rule: for nested SSH identity checks, run literal hostname commands without local command substitution. + +## FORUM_PROD_VM160_SWAP_PROOF_QUOTING_20260701 +- Context: VM160 swapfile proof 429. +- Issue: nested command substitution expanded on pve02, producing invalid SWAPFILE/FSTAB proof lines. +- Impact: proof 429 is not valid closure evidence even though swap was active. +- Rule: avoid nested $(...) in VM SSH proofs; use literal remote commands and clean proof. + +## FORUM_UPLOAD_ARCHIVES_VM_VERIFY_QUOTING_20260701 +- Context: proof 446 copy/check archives inside VM160. +- Issue: nested SSH quoting expanded shell variables incorrectly, producing gzip checks against empty .gz and blank TAR_TOP lines. +- Impact: SCP_TO_VM_RC=0 and file size list are valid, but GZIP/TAR proof lines in 446 are invalid. +- Rule: verify VM archive integrity locally inside forum-prod, not through nested quoted loops. + +## FORUM_PROD_BULK_IMPORT_PHP85_EMPTY_FRONTEND_20260701 +- Context: rebuilt VM160, five XenForo forums imported in bulk on PHP 8.5.7. +- Symptom: all five forums returned HTTP 200 with empty body; XenForo logged Template public:PAGE_CONTAINER/forum_list is unknown. +- Additional failed approach: proof 480 ran xf-dev:rebuild-caches on CodeVipe and returned RC=1, not a valid fix. +- Decision: stop bulk repair, treat current VM160 as lab, find a working CodeVipe recipe first, then destroy/rebuild cleanly for all five forums. +- Rule: no more mass rebuild/repair commands across all forums until single-forum CodeVipe recipe is proven. + +## XENFORO_REQUIREMENTS_DIRECT_DOWNLOAD_403_20260701 +- Context: Tried to download XenForo requirements ZIP directly from xenforo.com inside forum-prod. +- Issue: curl returned HTTP 403; unzip then failed because the ZIP was not downloaded. +- Evidence: proof 491 showed DOWNLOAD_RC=22, ZIP_LIST_RC=9, UNZIP_RC=9. +- Impact: proof 491 is not a valid server compatibility test. +- Rule: use operator-uploaded xenforo23-requirements-test.zip from /root/forum-upload instead of relying on direct curl download. + +## PVE01_UNZIP_MISSING_FOR_XF_REQUIREMENTS_VERIFY_20260701 +- Context: proof 492 tried to inspect uploaded xenforo23-requirements-test.zip on pve01. +- Issue: pve01 does not have unzip installed, so ZIP_TEST_RC=127 was not a ZIP integrity result. +- Impact: proof 492 confirmed file presence only, not archive validity. +- Rule: verify ZIP with Python zipfile or inside forum-prod rather than installing unzip on Proxmox just for inspection. + +## FRESH5_DEPLOY_SUCCESS_20260701 +- Context: VM160 was destroyed/recreated, then five XenForo forums were deployed from fresh ZIP backups. +- Result: proof 513 confirms all five forums locally healthy. +- Note: proof 511 initially failed dsmods path detection; proof 512 fixed dsmods by locating public_html via src/XF.php marker. +- Rule: future restores should preserve internal_data/code_cache or rebuild compiled template cache before smoke testing. + +## NPMPLUS_SQLITE_PASTE_FAILURE_20260701 +- Context: NPMplus SQLite inspection command was too complex and a Python fragment was pasted into bash. +- Issue: shell entered multiline prompt and produced syntax errors. +- Impact: do not trust that SQLite inspection attempt. +- Rule: avoid nested SSH heredoc/Python for this task; use simpler commands, generated nginx configs, or uploaded/local script files. + +## NPMPLUS_PUBLISH_SCRIPT_WRONG_ENV_SOURCE_525_20260701 +- Context: proof 525 tried to create forum proxy hosts in NPMplus. +- Issue: script sourced /opt/npmplus/.env, but INITIAL_ADMIN_EMAIL/PASSWORD were not present there; they are in the npmplus container env. +- Impact: no forum proxy hosts were created by proof 525. +- Rule: read NPMplus API login values from docker inspect env internally, never print them. + +## NPMPLUS_API_LOGIN_MISSED_HTTPS81_AND_JSON_ESCAPE_20260701 +- Context: NPMplus API login attempts in proofs 526/527 failed. +- Issue: commands missed the likely HTTPS admin endpoint on port 81 and built JSON by string interpolation instead of jq escaping. +- Impact: no proxy hosts were created by 526/527. +- Rule: for NPMplus API use jq-generated JSON and test https://127.0.0.1:81/api/tokens with -k before publish. + +## NPMPLUS_API_AUTH_UNAVAILABLE_MANUAL_ROUTES_20260701 +- Context: NPMplus API login attempts failed even though the forum VM and edge-to-forum connectivity are healthy. +- Issue: API credentials from container initial env are not accepted by current NPMplus API. +- Impact: do not use NPMplus API for this publish path. +- Rule: publish forum routes through backed-up manual nginx proxy_host configs, then issue proper certificates after DNS points to edge. + +## EDGE_CERTBOT_TOKEN_FILE_PERMISSION_535_20260701 +- Context: proof 535 attempted DNS-01 certificate issue for five forum domains. +- Issue: /tmp/forum_cf_token.env was root-owned mode 600, so debian user could not source it directly. +- Impact: certbot ran without token and all certificate attempts failed; DNS cutover must not proceed from proof 535. +- Rule: read Cloudflare token via sudo without printing it, then create temporary certbot credentials file and remove it after use. + +## EDGE_CERTBOT_FAILED_DNS_CUTOVER_ABORTED_20260701 +- Context: edge manual routes are working locally, but certbot DNS-01 certificate issue failed in proof 538. +- Issue: all five certbot attempts returned RC=1; proof 539 correctly aborted DNS cutover because edge cert precheck was not ready. +- Impact: Cloudflare DNS records were not changed by proof 539; public DNS still points to old IP 87.236.18.45. +- Rule: inspect certbot logs and Cloudflare token zone permissions before retrying certificate issuance or DNS cutover. + +## CLOUDFLARE_TOKEN_NO_ZONE_ACCESS_20260701 +- Context: Cloudflare token discovered on edge verified as a token but could not access forum zones. +- Evidence: proof 542 showed all five zones missing and DNS record create probes failed. +- Impact: certbot DNS-01 failed and DNS cutover was correctly aborted; public DNS still points to old hosting. +- Rule: install a new Cloudflare token with Zone Read and DNS Edit for the five forum zones before retrying certs/DNS. + +## FORUM_PUBLICATION_FINAL_SUCCESS_20260701 +- Context: five XenForo forums were rebuilt on VM160, routed through edge NPMplus, issued Let’s Encrypt certs using a corrected Cloudflare token, and cut over in DNS. +- Result: final public proof 546 passed. +- Important corrections: initial NPMplus API route failed; manual Nginx routes were used. Initial Cloudflare token had no zone access; corrected token passed zone/DNS probe. + +## FINAL_SNAPSHOT_NAME_TOO_LONG_556_20260701 +- Context: proof 556 final health gate passed for all five public forums. +- Issue: Proxmox snapshot failed because generated snapshot name exceeded 40 characters. +- Evidence: qm snapshot returned snapname value may only be 40 characters long. +- Impact: forum health was OK, but proof 556 snapshot step was not completed. +- Fix: rerun snapshot with short name. + +## CLOUDFLARE_DNS_AUDIT_TXT_SUBSTRING_GAP_20260701 +- Context: prior Cloudflare DNS audit checked web A/AAAA/CNAME state for root/www records. +- Issue: it did not deeply inspect TXT/SPF contents for old provider IP substrings. +- Evidence: user screenshot showed SPF TXT containing old 87.236.18.* value. +- Impact: web routing is OK, but mail-related DNS may still contain stale provider data. +- Rule: perform full DNS record audit including TXT/SPF/DKIM/DMARC/MX/CDN and substring search for old IPs before DNS cleanup. + +## CLOUDFLARE_DNS_CLEANUP_DRYRUN_QUOTE_563_20260701 +- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt. +- Impact: proof 563 is invalid and no DNS cleanup was performed by it. +- Rule: use copied scripts for Cloudflare DNS mutation instead of fragile inline nested quotes. + +## SPF_DUPLICATE_AFTER_565_20260701 +- Context: SPF cleanup command 565 attempted to replace stale SPF records. +- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained. +- Impact: domains temporarily had duplicate SPF records, which is invalid for mail validation. +- Fix: delete all SPF TXT records for the five forum zones, then create exactly one v=spf1 -all TXT per zone. + +## MSMTP_SECRET_SOURCE_PARSE_ERROR_576_20260701 +- Context: proof 576 installed msmtp but sendmail auth test failed. +- Issue: [SENSITIVE_PATH] was sourced as shell, but SMTP password contained shell-special characters; source failed and auth used an invalid/truncated secret path. +- Impact: msmtp package installed, but mail sending was not proven working. +- Rule: never source SMTP secret files containing arbitrary passwords; store password base64 and use msmtp passwordeval helper accessible to www-data. + +## FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701 +- Context: attempted to fix msmtp config with passwordeval helper. +- Issue: one or more sendmail/PHP mail tests still failed. +- Impact: XenForo mail sending is not yet proven. +- Next step: verify SMTP credential/password and provider policy for noreply@pvepro.ru. + +## SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701 +- Context: SMTP password was exposed in terminal output during failed msmtp setup. +- Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line. +- Impact: treat that SMTP password as compromised. +- Required action: rotate the mailbox/app password in the mail provider panel before any further SMTP testing. +- Rule: never store arbitrary passwords as shell-sourceable KEY=value files; use non-printing secret storage only. + +## NETBIRD_MAILCOW_REPORTED_DOWN_AFTER_SMTP_INCIDENT_20260701 +- Context: user reported NetBird and Mailcow became unavailable after forum mail-transport work. +- Impact: treat as active incident until service reachability and container/VM state are proven. +- Rule: do not change DNS, firewall, proxy, mail, NetBird, or Mailcow configs before read-only triage proof. + +## FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701 +- Context: forum-prod SMTP/msmtp test used invalid or compromised credentials and triggered Mailcow netfilter warnings/ban for 95.84.154.183. +- Impact: forum mail transport is not proven and must remain disabled until SMTP password is rotated and tested once carefully. +- Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth. +- Rule: do not test SMTP auth again until a fresh rotated mailbox/app password is available. + +## FORUM_OLD_CODEVIPE_CLOUD_BACKUP_STALE_AFTER_FRESH5_20260701 +- Context: pve02 codevipe-cloud-backup.timer existed before fresh five-forum rebuild. +- Issue: old script points to /root/backups/codevipe and old remote path app/codevipe/forum-prod/local. +- Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory contains old codevipe-public/codevipe.sql files, not current five-forum archive. +- Impact: old timer must not be treated as valid current backup for all five forums. +- Fix: create a new forum-prod fresh5 cloud backup using existing pve02 Mail-cloud rclone crypt remotes. + +## FORUM_RESTORE_DRILL_610_STATUS_FLAG_BUG_20260701 +- Context: restore drill proof 610 downloaded Mail.ru Cloud backup, verified split part SHA256, reconstructed archive, matched full archive SHA256, and confirmed all five forum DB/file archives in tar. +- Issue: final RESTORE_DRILL_OK was set to no because the script tested an unassigned shell variable RCLONE_REMOTE_PVE02_MAIL_01_RC. +- Impact: proof 610 final status flag is invalid, but its detailed integrity checks are valid. +- Fix: rerun corrected restore validation against the downloaded cloud backup artifacts and produce proof 612. + +## FORUM_XENFORO_MAIL_SMOKE_TEST_FAILED_20260701 +- XenForo-level mail smoke test did not return success for all five forums. +- Check proof 623 and msmtp log before retrying. + +## XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701 +- Context: custom XenForo mail smoke test 623 failed with Could not open input file /root/scripts/623_xenforo_mail_smoke.php. +- Cause: the script was run as www-data from /root/scripts; /root is not traversable by www-data. +- Impact: proof 623 is invalid and should not be used to judge mail delivery. +- Superseding evidence: user ran XenForo built-in outgoing email test; message was delivered to Yandex but placed in Spam. +- Next step: investigate deliverability/authentication headers and pvepro.ru SPF/DKIM/DMARC/PTR. + +## FORUM_NEXT_CHAT_KNOWN_ERRORS_AND_CLOSED_INCIDENTS_20260701 + +### Closed / classified incidents +- SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701: + - A previous bad command sourced a raw SMTP secret and printed it. + - Treat old password as compromised. + - Later persistent SMTP was rebuilt using safe files and verified. + - Never print or package secrets. + +- XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701: + - Custom mail proof 623 failed with "Could not open input file". + - Cause: PHP was run as www-data from /root/scripts, but /root is not traversable by www-data. + - Impact: proof 623 is invalid and must not be used to judge mail delivery. + - Superseded by user-observed built-in XenForo test and Yandex header proof. + +- FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701 / FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701: + - Earlier SMTP attempts failed and triggered mailcow netfilter ban for forum public IP. + - Ban was removed. + - Persistent msmtp transport was later enabled and verified. + - Mailcow and NetBird remained reachable after final tests. + +- SPF_DUPLICATE_AFTER_565_20260701: + - Earlier SPF cleanup created duplicate SPF records. + - Fixed by deleting duplicates and recreating exactly one SPF per forum domain. + - Final forum domains use v=spf1 -all because sending uses noreply@pvepro.ru. + +- FORUM_RESTORE_DRILL_610_STATUS_FLAG_BUG_20260701: + - Restore drill 610 had a status flag bug despite successful detailed checks. + - Corrected restore drill 612 passed. + +- FORUM_DBTECH_TOR_TIMEOUT_CLASSIFIED_20260701: + - One historical timeout row per forum to check.torproject.org. + - DBTech/Security active, but dbtech_security_tornodes=0. + - Classified as external timeout noise, not runtime failure. + +### Current non-blocking items +- Yandex placed built-in XenForo test mail in Spam despite SPF pass and DKIM pass. +- Classification: deliverability/reputation/content filtering, not server failure. +- Recommended manual action: click “Не спам!” and add/check pvepro.ru in Yandex Postmaster. + +### Safety rules for next chat +- Do not print secrets. +- Do not download or upload: + - [SENSITIVE_PATH] + - /etc/msmtprc + - /etc/msmtp/* + - rclone configs + - Cloudflare tokens + - DB dumps + - VM disks + - backup archives +- For handoff, only share the two truth files and selected non-secret proof files. + +## PARKED_DOMAINS_STAGE4_DNS01_PREFLIGHT_FAILED_20260701 +- Context: parked-domain public apply proof 634. +- Issue: DNS-01 based apply was attempted before a clear CAN_APPLY=yes preflight result was visible in terminal output. +- Observed: stage4 stopped safely with preflight_failed; CERTBOT_PRESENT=no, CERTBOT_DNS_CLOUDFLARE_PLUGIN=no, and CF_ZONE_ACCESS=no for newfi.ru, hapusya.ru and kingofwolk.ru. +- Impact: no parked-domain NPMplus route/certificate changes were applied by stage4; public HTTPS validation still failed certificate verification. +- Rule: do not use Cloudflare DNS-01 for these parked domains unless a token with explicit zone access is installed; because DNS already points to edge, prefer a copied-script HTTP-01 flow with temporary ACME challenge routes and content-specific validation. + +## PARKED_DOMAINS_STAGE5_BASH_LOCAL_SETU_BUG_20260701 +- Context: parked-domain HTTP-01 apply proof 635. +- Issue: edge script used Bash `local id="$1" ... conf="$WORK/.../$id.conf"` and `local host="$1" ... tmp="$WORK/.../$host.html"` under `set -u`; dependent variables are not safe inside the same local assignment command. +- Observed: `id: unbound variable`, `source can not be empty`, then `host: unbound variable`; certbot did not run, final HTTPS routes were not installed. +- Impact: local parked page remained OK; public HTTPS remained not closed. +- Rule: split dependent Bash local assignments into separate statements, syntax-check copied scripts before execution, and keep content-specific validation for parked routes. + +## PARKED_DOMAINS_STAGE6_TEMP_HTTP_ROOT_VALIDATION_BUG_20260701 +- Context: parked-domain HTTP-01 fixed apply proof 636. +- Issue: temporary HTTP-01 route validation checked HTTP `/` for parked marker, but expected behavior before certificates is root HTTP 301 to HTTPS. +- Observed: LOCAL_HTTP_VALIDATE returned HTTP_CODE=301 MARKER_OK=no, script rolled back its temporary route configs and did not run certbot. +- Impact: local parked page remained OK; public HTTPS remained not closed. +- Rule: validate `/.well-known/acme-challenge/` for HTTP-01 readiness, not HTTP `/`; also scan all active NPMplus nginx config dirs for unmanaged domain conflicts before writing routes. + +## PARKED_DOMAINS_STAGE7_NEWFI_ROOT_ACME_REDIRECT_20260701 +- Context: parked domains HTTP-01 Stage7 stopped before certbot because ACME validation path returned 301 for root newfi.ru while www.newfi.ru, hapusya.ru, www.hapusya.ru, kingofwolk.ru and www.kingofwolk.ru returned 200. +- Impact: no final parked-domain HTTPS routes/certificates were applied by Stage7; temporary route configs were rolled back. +- Rule: do not run another apply stage until active NPMplus nginx route selection for newfi.ru is diagnosed with nginx -T and Host-header probes. + +## PARKED_DOMAINS_STAGE8_DEFAULT_HTTP_TO_HTTPS_REDIRECT_20260701 +- Context: parked-domain route autopsy proof 638. +- Finding: no active NPMplus config contains newfi.ru/hapusya.ru/kingofwolk.ru, but unknown HTTP hosts and ACME paths return 301 to HTTPS from NPMplus default routing. +- Impact: HTTP-01 cannot work through the default server. A dedicated temporary ACME server block must be proven before certbot; do not run another certbot apply without a successful ACME-path probe for all six hostnames. +- Rule: after every parked-domain temp route attempt, rollback the route and prove rollback before continuing. + +## PARKED_DOMAINS_STAGE9_NEWFI_EXACT_PROBE_NEEDED_20260701 +- Context: parked domains HTTP-01 probe stage9 installed one combined temporary server block for all six hostnames. +- Issue: five hostnames returned ACME probe 200, but root newfi.ru still returned 301. +- Impact: do not continue certbot/apply until exact newfi.ru route behavior is isolated. +- Rule: run a temporary exact newfi.ru probe block and roll it back before any HTTP-01 apply. + +## PARKED_DOMAINS_STAGE10_EXACT_NEWFI_PROBE_OK_20260701 +- Context: parked-domain Stage10 proof 640. +- Result: exact server block for newfi.ru on NPMplus intercepted both root and ACME paths with HTTP 200 and custom X-Parked-Probe markers. +- Stage9 combined server_name block passed 5/6 but failed bare newfi.ru, so final HTTP-01 cutover must use separate exact per-host server blocks. +- Rule: do not use one combined parked-domain server_name block for these domains; use exact host blocks and content-specific validation. + +## PARKED_DOMAINS_STAGE11_LOW_ID_EXACT_ROUTE_FAILED_20260701 +- Context: parked-domain Stage11 proof 641. +- Result: exact per-host route files 700-705 passed nginx -t but bare newfi.ru still hit HTTP->HTTPS redirect on ACME path. +- Earlier Stage10 proof showed exact newfi.ru route works when placed at high id 998. +- Hypothesis: file/include priority or NPMplus generated ordering makes low id 700 unsuitable for bare newfi.ru in this runtime. +- Rule: do not retry parked-domain HTTP-01 with low id 700-705. First prove fixed high-id exact blocks, then apply with those exact ids. + +## PARKED_DOMAINS_SWITCH_TO_DNS01_AFTER_HTTP01_FAILURES_20260701 +- Context: parked domains newfi.ru, hapusya.ru and kingofwolk.ru. +- HTTP-01 attempts hit NPMplus default HTTP-to-HTTPS redirect/include-order problems. +- Stage12 cleanup confirmed temp files 980-985 absent and nginx reload OK. +- Rule: do not continue HTTP-01 for these parked domains; use DNS-01 with a dedicated Cloudflare token that has Zone Read and DNS Edit for the three zones. + +## PARKED_DOMAINS_STAGE15_FALSE_LOCAL_VALIDATE_ROLLBACK_20260701 +- Context: Stage15 DNS-01 succeeded and public validation showed parked page for most hosts, but local edge curl --resolve validation returned CURL_RC=60 and triggered rollback. +- Impact: do not rerun Stage15 as-is. +- Rule: for parked domains finalization, use existing DNS-01 certificates and validate from pve01 public HTTPS with content marker; do not use the failed local --resolve TLS check as the closure gate. + +## DOMAIN_ACTIONS_STAGE18_PLACEHOLDER_AND_RENEWAL_PROOF_20260701 +- Context: after parked domains and portfolio audit, operator requested removing domain enumeration from placeholder and checking certificate renewal paths. +- Rule: Stage18 must not change DNS records or NPMplus route files. It may update placeholder HTML, run existing forum renewal script once for runtime proof, and run certbot dry-run for parked certificates using existing DNS-01 hooks. + +## GRAM1_ROOT_WWW_PLACEHOLDER_CLOSE_STAGE19_20260701 +- Context: operator requested gram1.ru root/www to use the existing placeholder page. +- Rule: only root gram1.ru and www.gram1.ru may be routed. Existing gram1.ru subdomain NPMplus routes must not be changed. +- Rule: use DNS-01 with a dedicated gram1 Cloudflare token; do not use HTTP-01. +- Stage18 parked dry-run note: kingofwolk.ru dry-run hit Let's Encrypt rateLimited/service-busy after active certificate issuance and public HTTPS were already OK. Treat as transient external dry-run failure, not route/cert closure blocker. + +## PVEPRO_EDGE_LANDING_STAGE21_20260701 +- Context: pvepro.ru root/www currently point to the Mailcow VPS and return Mailcow with TLS hostname mismatch. +- Decision: because pve01 has no SSH key access to the external VPS, do not modify the VPS. Instead move only pvepro.ru and www.pvepro.ru A records to edge and serve an edge landing page. +- Safety rule: do not change mail.pvepro.ru, nb.pvepro.ru, MX, SPF, DKIM, DMARC or forum SMTP configuration. +- Use a dedicated Cloudflare token for pvepro.ru. Do not print token values. + +## PVEPRO_STAGE21_LANDING_SERVICE_FAILED_20260701 +- Context: Stage21 stopped safely before DNS/cert/NPMplus route because local pvepro landing service returned CURL_RC=7 on 127.0.0.1:18089. +- Impact: DNS root/www pvepro.ru remained on 185.139.214.215 and no route/cert changes were applied. +- Rule: Stage22 must fix and prove the local landing service first, then proceed to DNS-01 cert, NPMplus route and root/www DNS only after local marker PVEPRO_LANDING_OK is reachable. + +## PVEPRO_STAGE22_VALIDATION_AND_COPY_GUARD_20260701 +- Context: Stage22 succeeded on edge-side landing, cert, NPMplus route and Cloudflare A upserts, but wrapper had a non-critical scp wildcard failure due __pycache__ and public validation used cached DNS for root pvepro.ru. +- Mistakes recorded: do not scp scripts/* after local py_compile because __pycache__ may be present; do not continue after critical scp failure in future apply stages; do not classify PVEPro landing as NetBird just because it contains a link text; use Cloudflare authoritative A records and --resolve validation when DNS propagation may lag. +- Stage23 must be read-only for infrastructure and may close only if edge route, Cloudflare A records, HTTPS landing marker, and mail/nb unchanged checks pass. + +## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701 +- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router. +- Current limitation: there is no safe private remote access path to the dacha router yet. +- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet. +- Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists. + +## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701 +- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router. +- Current limitation: there is no safe private remote access path to the dacha router yet. +- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet. +- Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists. + +## TAFTAUTO_WG_STAGE24_SECRET_AND_IMPORT_RULE_20260701 +- Context: configuring private management path for taftauto.ru dacha router. +- Rule: do not print WireGuard private keys, PSK, or client config body in proof/chat. +- Rule: generated client config may be stored only as root/debian mode 600 file and must be imported into the Netcraze router UI manually. +- Rule: do not open router public admin or certificate deployment until WireGuard handshake is proven. + +## TAFTAUTO_WG_PSK_ROTATION_SCRIPT_BROKE_DATAPLANE_20260702 +- Assistant gave unsafe PSK rotation flow; after profile reimport dacha interface changed from Wireguard0 to Wireguard1 and data-plane broke until security-level, ACL and routes were restored on Wireguard1. +- Proof 666 is superseded because it showed PUBLIC_SSH_22_STILL_OPEN. +- Final fixed proof: /root/evidence/667_TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702_PROOF.txt + +## TAFTAUTO_CERTBOT_CLOUDFLARE_PLUGIN_MISSING_20260702 +- Certbot on edge rejected --dns-cloudflare-* arguments: dns-cloudflare plugin missing/not loaded. +- No taftauto certificate issued in failed run; no DNS A records or NPMplus routes changed. + +## HOMEPAGE_SERVICES_YAML_BAD_INDENT_20260702 +- Assistant inserted Homepage services.yaml entries with wrong indentation; YAMLException bad indentation at line 227. +- Fix: rebuild SmartApe under External Homelabs with 2-space item indentation, remove Router Moscow, rebuild Sites category. + +## HOMEPAGE_SERVICES_YAML_BAD_INDENT_REPAIR_20260702 +- Previous apply broke services.yaml indentation and did not follow YAML-aware rule. +- Repair uses latest valid services.yaml backup, PyYAML parse/dump, validates before restart. + +## TAFTAUTO_CERTBOT_DRYRUN_BAD_FLAG_20260702 +- Assistant used unsupported certbot flag --manual-public-ip-logging-ok with certbot 4.0.0 during taftauto renew dry-run. +- No certificate, DNS, NPMplus route, or deploy state changed by the failed dry-run. + +## TAFTAUTO_CERT_RENEW_DRYRUN_RATE_LIMITED_20260702 +- Proof 672 is superseded/partial: it printed STATUS=OK even though certbot dry-run failed. +- Failure reason: Let's Encrypt staging/service returned rateLimited / Service busy; retry later. +- Cloudflare manual auth and cleanup hooks did run successfully. +- Deploy hook was installed and manually invoked successfully before the dry-run. +- Do not retry immediately. + +## HOMEPAGE_XENFORO_INFO_INLINE_PYTHON_SYNTAX_ERROR_20260702 +- Assistant used invalid inline Python syntax with def after semicolon while adding XenForo.info to Homepage External. +- Failed before services.yaml write. + +## HOMEPAGE_USEFUL_ROUTER_NAME_MISMATCH_20260702 +- Previous apply looked for Russian Router card name, but active Homepage Useful group contains Router and Public Domain. +- Failed before write. Fix uses YAML-aware edit with exact names Router/Public Domain. + +## HOMEPAGE_USEFUL_ROUTER_CARD_NOT_FOUND_20260702 +- Apply expected Useful -> Router in active services.yaml, but YAML parse did not find it. No write was performed. +- Need verify active Homepage config path, docker mount, and any alternate services.yaml before next apply. + +## 20260702_CF_TOKEN_AUDIT_BROKEN_COMMAND +- Ошибка: ассистент дал хрупкую команду аудита Cloudflare-токенов с вложенным heredoc/Python через ssh. +- Факт: bash начал интерпретировать Python-код, появились syntax error и вторичный prompt >. +- Правило: не давать nested heredoc/сложные кавычки через ssh; для длинной логики сначала класть скрипт отдельным файлом, затем запускать его. +- Статус: аудит токенов не выполнен, инфраструктурные изменения не должны считаться применёнными. + +## LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE +- Do not treat e3qxxx.netbird.selfhosted / 100.100.125.70 and e3qxxx-183-106.netbird.selfhosted / 100.100.183.106 as VPS egress nodes; they are user mobile peers. +- Correct egress candidates are relay.pvepro.ru / relay.netbird.selfhosted / 100.100.19.1 for Moldova and mail.pvepro.ru / mail.netbird.selfhosted / 100.100.147.204 for USA. +- Access Policy edge-vm -> relay/mail only proves peer reachability. It is not an internet exit route. +- Required NetBird fix for CrowdSec CAPI was Network Routing exit route 0.0.0.0/0 distributed to edge-vm group, routing peer relay, masquerade/NAT enabled. +- Before attempting CrowdSec CAPI, verify on edge-vm: ip route get 1.1.1.1 uses wt0, wg allowed-ips contains 0.0.0.0/0, and Cloudflare trace no longer shows home IP 95.84.154.183. + +## LESSON_20260702_HOMEPAGE_MONITORS_REPAIR_NOT_DELETE +- Do not delete or disable Homepage siteMonitor fields to hide red badges. +- Fix the underlying monitor path, DNS, proxy route, or local health-wrapper instead. +- Do not touch Cloudflare Homepage card when operator says it is green and opens correctly. +- Before changing Homepage, identify the live config host; in this state live Homepage is on core-apps [PRIVATE_IP], not edge-vm. + +## LESSON_20260702_KUMA_ADD_ONE_MONITOR_ONLY +- When asked to fix one missing Uptime Kuma service, add only one monitor, then generate proposals separately. +- Before direct Kuma DB mutation, stop the container and create a DB backup. +- Verify DB integrity before starting Kuma again. +- Do not touch Cloudflare when operator says it is green and opens correctly. + +## LESSON_20260702_DOCKGE_STALE_STACKS_NOT_CONTAINERS +- Dockge inactive items after migration can be stale compose folders, not stopped containers. +- First classify runtime projects across all Docker hosts before deleting or archiving anything. +- For remote Dockge visibility, projects must be under /opt/stacks or bind-mounted there. +- Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions. + +## STAGE4B_MASKED_SQL_QUERY_FAILURE_20260714 +- Previous Stage4B read-only audit returned empty observation-summary, metadata-key, trigger and function blocks because nested SSH/SQL quoting broke queries while stderr was hidden. +- Empty blocks are query failures, not proof that metadata, triggers or functions are absent. +- Future PostgreSQL audits must use ON_ERROR_STOP, visible stderr, independent query RC and quote-safe dollar literals. + +## STAGE4C_SCHEMA_MIGRATIONS_ID_ASSUMPTION_20260714 +- Stage4C read-only preflight incorrectly assumed schema_migrations has a column named id. +- PostgreSQL returned column id does not exist and ON_ERROR_STOP correctly stopped all following SQL checks. +- Production DB and application were not changed. Future migration audits must inspect information_schema first and query migration rows without assuming column names. + +## STAGE4C_PGDUMP_DEV_NULL_FSYNC_INVALID_PROBE_20260714 +- Stage4C custom-format pg_dump capability probe incorrectly used /dev/null as the output file. +- pg_dump failed only because fsync on /dev/null returned Invalid argument. +- Production database and application were not changed. +- Future custom dump capability checks must use a real temporary regular file, validate it with pg_restore --list, and remove it afterward. + +## STAGE4C_REMOTE_PREP_THIRD_SUBSTEP_FAILED_20260714 +- Isolated migration dry-run did not start: REMOTE_PREP_RC=0|0|1. +- Remote directory creation and SCP succeeded; the third ownership/mode preparation substep returned RC=1. +- Production database and application were not changed. +- Exact cause requires read-only residual directory and permission inspection before cleanup or retry. + +## STAGE4C_REMOTE_PREP_GLOB_AFTER_CHMOD_ROOT_CAUSE_20260714 +- Exact cause of REMOTE_PREP_RC=0|0|1: after the directory became postgres-owned mode 0700, the unprivileged debian shell could not traverse it and could not expand the wildcard used by chmod. +- Candidate files and SHA256 verification were valid; no temporary database was created and production was unchanged. +- Future preparation must use exact privileged paths or a verified shared group, never an unprivileged wildcard after restrictive chmod. + +## STAGE4C_MIGRATION003_FINGERPRINT_DOLLAR_QUOTE_FAILURE_20260714 +- Migration 003 isolated dry-run failed at fingerprint CHECK because the regex end anchor collided with the SQL dollar-quote delimiter and produced an unterminated dollar-quoted string. +- The migration transaction rolled back, the temporary database was removed, and production remained unchanged. +- Fingerprint validation must use length plus translate without a regex dollar anchor; corrected candidate must pass a fresh isolated dry-run before production consideration. + +## STAGE4C_SEAL_OUTER_RC_MASKING_20260714 +- Stage4C final seal correctly refused creation because overall health was WARN, returning internal RC=40. +- The former wrapper then printed a second misleading COMMAND_RC=0 because top-level true masked the internal result. +- Future commands must capture the guarded subshell through PIPESTATUS and print exactly one authoritative COMMAND_RC before the mandatory final true. +- Production database, application and services were unchanged. + +## STAGE4C_DIAGNOSIS_EXPECTED_DERIVED_FILE_ON_VM180_20260714 +- Previous diagnosis incorrectly treated cluster-admin-restricted-probes.txt as a required source file on VM180. +- The file is a central derived health artifact on pve01; VM180 supplies full-observer evidence instead. +- This false expectation caused diagnostic RC=32 without any infrastructure failure or production change. + +## STAGE4C_NO_JOBS_RUNNING_TEXT_COUNTED_AS_JOB_20260714 +- Stage4C seal preflight incorrectly counted the systemctl informational text No jobs running as one active job. +- The parser counted every nonempty output line instead of accepting only rows beginning with a numeric systemd Job ID. +- No service was started and no production state changed during the blocked attempt. +- Future job counts must match a numeric first field only. + +## STAGE4D_ADAPTER_SOURCE_AUDIT_ABORTED_BEFORE_FINAL_MARKERS_20260714 +- The evidence-only Stage4D adapter source audit returned RC=1 before emitting its VM180 completion and final audit markers. +- The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults. +- Production, database, application, services, timers, health and desired-state were unchanged. +- Inspect the preserved runner stdout, stderr and result JSON before correcting or rerunning the task. + +## STAGE4D_LOCAL_EXPANSION_OF_REMOTE_Q_UNDER_NOUNSET_20260714 +- The first Stage4D adapter source audit embedded a large remote program inside a locally double-quoted SSH argument. +- Local shell expansion reached the remote variable Q while nounset was enabled, causing Q: unbound variable before SSH execution. +- VM180 and PostgreSQL audits did not start; production state was unchanged. +- Remote shell and SQL programs must be separate syntax-checked files passed through SSH stdin. + +## STAGE4D_REMOTE_PYTHON311_ADAPTER_SELFTEST_RC1_20260714 +- Canonical adapter candidate passed local Python 3.13 self-tests with 48 envelopes but returned RC=1 on VM180 Python 3.11.2. +- Remote upload and cleanup succeeded, and production database remained 0|0|OK. +- The exact traceback and failing assertion must be captured by an isolated identical reproduction before changing the candidate. + +## STAGE4D_PYTHON311_FSTRING_COMPATIBILITY_AND_REPRO_VALIDATOR_20260714 +- Adapter candidate passed Python 3.13 but failed Python 3.11 parsing at Path(row["path"]) inside a double-quoted f-string. +- Exact fix is Python 3.11-compatible quoting: Path(row['path']). +- The reproduction command captured the expected compile failure but incorrectly returned CAPTURE_BAD=1 because its validator required compile RC=0. +- Production database remained 0|0|OK and desired-state remained clean. +- Future adapter candidates must pass syntax and full self-tests on both pve01 and VM180 runtimes before acceptance. + +## STAGE4E_DESIGN_VALIDATOR_ABORTED_BEFORE_LOCAL_VALIDATION_MARKER_20260714 +- Restricted probe-agent design candidate compiled successfully but its local validator exited before producing LOCAL_VALIDATION_RC. +- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults. +- Active external probes were not executed and production state was unchanged. +- Inspect the preserved validator traceback and exact assertion before modifying the candidate. + +## STAGE4E_VALIDATOR_SCANNED_PYC_AND_OWN_FORBIDDEN_LITERALS_20260714 +- Stage4E design validator recursively read every candidate file as UTF-8 after py_compile had created binary __pycache__ bytecode. +- This caused UnicodeDecodeError before any design assertion failed. +- The phase-wide forbidden-pattern scan also included validate_design.py itself, which necessarily contains the prohibited literals it verifies. +- Validators must use an explicit text-file allowlist and scans must exclude validator implementation and binary artifacts. +- Production, database, services, timers and desired-state were unchanged; active external probes were not executed. + +## STAGE4E_V2_VALIDATOR_FAILED_STATIC_COMPILE_20260714 +- Stage4E design candidate v2 failed static compilation before local design validation started. +- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects. +- The generated validator must be inspected at the exact SyntaxError line before another candidate is created. +- Production, database, application, services, timers, health and desired-state were unchanged; active external probes were not executed. + +## STAGE4E_PIPESTATUS_LOST_AFTER_FIRST_ASSIGNMENT_20260714 +- VM180 validation copy used a pipeline and read PIPESTATUS in two separate assignments. +- The first assignment reset PIPESTATUS, leaving COPY_RC empty and preventing remote validation and cleanup. +- Retry must use explicit SCP operations without pipeline status parsing. +- Production remained unchanged and active external probes were not executed. + +## STAGE4F_VALIDATOR_EXPECTED_HEALTH_REFRESH_FROM_NONWRITING_BACKUP_SCRIPT_20260714 +- The controlled backup service completed with Result=success and ExecMainStatus=0. +- Validation incorrectly required cluster-admin-incident-engine-backup.txt to receive a fresh timestamp. +- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure. +- Stage4F backup proof must use the service result, fresh latest JSON, produced artifacts and off-host/restore evidence instead. +- The backup must not be rerun merely to satisfy the invalid health-file freshness assertion. + +## STRICT_RULE_20260714_CLOSE_TAILS_IMMEDIATELY +- Failure class: переход к следующей задаче при наличии незакрытого хвоста. +- Mandatory anti-regression: после ошибки запрещено продолжать другой scope до точной root cause, исправления, повторной проверки, очистки, proof и seal. +- Closure gate: UNRESOLVED_TAIL_COUNT=0; BLOCKER_COUNT=0; TEMPORARY_ARTIFACT_COUNT=0; ROLLBACK_VERIFIED=YES; HEALTH=OK; PROOF_UPDATED=YES; REFERENCE_UPDATED=YES. +- Если закрытие невозможно из-за внешней зависимости, опасной неоднозначности или обязательного решения пользователя, фиксировать BLOCKED/OPEN и не заявлять CLOSED. +- Proof: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/report.txt + +## ERROR_20260714_DEPENDENCY_FACT_QUERY_GLOBAL_LIMIT +- Symptom: dependency audit вернул только latest_collector_status. +- Root cause: ORDER BY 1 LIMIT 1 в конце UNION ALL применился ко всему набору фактов. +- Correction: latest collector status извлекается скалярным подзапросом внутри CTE facts; глобальный LIMIT отсутствует. +- Anti-regression: multi-fact SQL audit обязан проверять точное ожидаемое число строк до извлечения значений. +- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt +- Status: CLOSED. + +## ERROR_20260714_COLLECTOR_PATCH_REQUIREMENT_PATTERN_MISMATCH +- Symptom: MIGRATION_PLAN_COLLECTOR_PATCH_REQUIRED_COUNT=1 при ожидаемых двух подтверждениях. +- Root cause: узкий шаблон учитывал collector_patch_required, но не collector_patch_required_after_stage4c. +- Correction: оба поля проверяются отдельно, затем складываются как semantic requirement count. +- Anti-regression: связанные JSON-контракты проверяются по собственным точным именам полей, а не одним унифицированным grep-шаблоном. +- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt +- Status: CLOSED; PRIOR_TAIL_UNRESOLVED_COUNT=0. + +## ERROR_20260714_PRIVILEGED_FILE_REDIRECTION_BEFORE_SUDO +- Symptom: bash reported Permission denied while counting collector.py lines. +- Root cause: `< collector.py` was opened by the unprivileged caller shell before sudo executed wc. +- Correction: run sudo wc -l collector.py without caller-side input redirection. +- Anti-regression: never combine sudo with `< protected-file`; pass the protected path as an argument to the privileged command. +- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt +- Status: CLOSED. + +## ERROR_20260714_COLLECTOR_RUN_COLUMN_ASSUMPTION +- Symptom: SQL failed because completed_at did not exist. +- Root cause: collector_runs columns were assumed instead of read from information_schema; actual names are finished_at and error_text. +- Correction: assert required and forbidden column counts before querying recent runs. +- Anti-regression: every schema-sensitive audit must discover and validate exact column names before the main query. +- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. + +## ERROR_20260714_REMOTE_STDIN_ARGUMENT_SHIFT +- Symptom: remote harness выполнил chmod для пути bash. +- Root cause: дополнительный positional token bash был передан после SSH command и стал первым аргументом stdin-скрипта. +- Correction: remote command вызывает bash -s -- с ровно тремя явно встроенными аргументами: candidate path, SHA256 и byte count. +- Anti-regression: stdin-скрипт обязан валидировать $1 по разрешённому path pattern и затем подтверждать точные SHA256 и byte count; запрещено добавлять отдельный bash-token после remote command. +- Closure proof: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/report.txt +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. + +## ERROR_20260714_OVERSIZED_COMMAND_PARSE_FAILURE +- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`. +- Root cause: чрезмерно длинная интерактивная команда была обрезана или повреждена при передаче до полного разбора Bash. +- Impact: side-effect audit подтвердил production state 0|0|OK, неизменный collector, активный timer и отсутствие временных БД, каталогов и процессов. +- Correction: монолитные команды запрещены; сложные действия выполняются через отдельный проверяемый task-скрипт и короткий launcher. +- Anti-regression: interactive payload target <= 8000 bytes; syntax-check перед запуском; remote output сохраняется независимо от RC. +- Closure proof: /var/lib/homelab-change-sets/operator-command-size-guard-v1/20260714T182653Z/report.txt +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. + +## ERROR_20260714_HOMELAB_ADMIN_HELP_RC_ASSUMPTION +- Symptom: template audit завершился HOMELAB_ADMIN_HELP_FAILED. +- Root cause: был ошибочно ожидаем RC=0 от неподдерживаемого аргумента --help. +- Actual contract: неизвестный аргумент печатает usage и возвращает RC=64. +- Correction: проверять version либо считать точную usage-строку вместе с RC=64 штатным результатом. +- Anti-regression: перед проверкой CLI сначала фиксировать поддерживаемые команды; не предполагать GNU-style --help. +- Closure proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v1/20260714T183338Z/report.txt +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. + +## ERROR_20260715_EXPECTED_NEGATIVE_RC_TRAP_CLASSIFICATION +- Symptom: schema-v2 task v3 build stopped while intentionally testing the --invalid argument path. +- Root cause: an expected RC64 was executed while the generic ERR trap remained active. +- Correction: expected failures are evaluated only through an explicit if/else branch with immediate RC capture. +ANTI_REGRESSION_TOKEN=EXPECTED_RC64_INTERCEPTED_ERR_TRAP +- Required pattern: if command; then rc=0; else rc=$?; fi, followed by a separate exact expected-RC assertion. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. + +## ERROR_20260715_DUPLICATE_MACHINE_MARKER_COUNT +- Symptom: error-register candidate construction stopped with RC1 before applying the candidate. +- Root cause: the same marker text appeared in both a heading and a descriptive line, while the guard expected one substring occurrence. +- Correction: headings and prose do not contain machine-token values; validation counts only complete exact token lines. +ANTI_REGRESSION_TOKEN=DUPLICATE_MACHINE_TOKEN_IN_HEADING_AND_BODY +- Required pattern: grep -Fxc against a complete machine line, never grep -Foc against an unrestricted substring. +- Production impact: none. +- Temporary artifacts: removed and verified. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T02:56:19Z + +## ERROR_20260715_OUTER_WRAPPER_SHELL_SYNTAX +- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis. +- Root cause: the outer wrapper contained an unbalanced compound shell construct before the closing subshell delimiter. +- Impact: parsing failed before the start marker, guards and homelab-admin invocation; no task phase or infrastructure action executed. +- Correction: use a linear wrapper with explicitly closed if/case blocks and avoid nested brace groups in conditional expressions. +ANTI_REGRESSION_TOKEN=OUTER_WRAPPER_UNBALANCED_SUBSHELL_SYNTAX +- Anti-regression: after any parser-level failure, treat the attempted phase as not executed and verify package and run-state before retrying. +- Production impact: none. +- Task package impact: none. +- Temporary artifacts: none. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T03:24:05Z + +## ERROR_20260715_INLINE_REMOTE_SHELL_IN_APPLY_PHASE +- Symptom: homelab-admin lint rejected Stage4H task v6 with ERR-002-INLINE-REMOTE-SHELL. +- Root cause: apply.sh invoked bash -s through an inline quoted SSH program and streamed the remote script through stdin. +- Correction: copy the declared remote script to an isolated fixed path on VM180 and invoke that file directly with positional arguments. +ANTI_REGRESSION_TOKEN=INLINE_REMOTE_SHELL_REPLACED_BY_STAGED_REMOTE_SCRIPT_PATH +- Anti-regression: phase scripts may perform transport orchestration, but remote shell program bodies must exist as separately declared, syntax-checked remote script files. +- Remote stdout and stderr must be preserved before evaluating the remote return code. +- Task v6 status: REJECTED_BY_LINT; never executed and never mutated. +- Production database impact: none. +- Live collector impact: none. +- Temporary database and remote root count after rejection: zero. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T04:13:46Z + +## ERROR_20260715_OPTIONAL_RUN_DIRECTORY_FIND_UNDER_PIPEFAIL +- Symptom: task-v7 builder stopped while counting formal runs for rejected task v6. +- Root cause: find was executed against an optional directory that did not exist; under pipefail the assignment returned RC1. +- Correction: test directory existence first and assign zero without invoking find when it is absent. +ANTI_REGRESSION_TOKEN=MISSING_OPTIONAL_RUN_DIRECTORY_TREATED_AS_ZERO +- Anti-regression: optional paths must have an explicit existence branch before find under pipefail. +- Production impact: none. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T04:44:06Z + +## ERROR_20260715_PG_CONSTRAINT_CONTYPE_CHAR_CONCAT +- Symptom: isolated Stage4H acceptance failed during schema baseline capture. +- Root cause: pg_constraint.contype uses the internal PostgreSQL char type and was concatenated without an explicit text cast. +- Correction: use contype::text or CAST(contype AS text). +ANTI_REGRESSION_TOKEN=PG_CATALOG_CHAR_CONCAT_REQUIRES_EXPLICIT_TEXT_CAST +- Machine rule: ERR-014-PG-CATALOG-CHAR-CONCAT. +- Negative self-test: uncast expression rejected with RC64. +- Positive self-test: explicit text cast accepted with RC0. +- Task v7 mutated: no. +- Production impact: none. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T06:27:06Z + +## ERROR_20260715_LIVE_COLLECTOR_VERIFIER_WRONG_PATH +- Symptom: verified linter installation was rolled back after the final collector integrity guard returned UNKNOWN. +- Root cause: verifier queried a nonexistent path instead of the canonical live collector path declared by the task acceptance script. +- Canonical path: /opt/cluster-admin-incident-engine/collector.py. +- Incorrect path: /opt/homelab-cluster-admin/cluster-admin-collector.py. +ANTI_REGRESSION_TOKEN=LIVE_COLLECTOR_VERIFIER_PATH_MUST_MATCH_TASK_CANONICAL_PATH +- Correction: derive and guard the live collector path from the immutable task contract before executing the hash query. +- Production impact: none; the canonical collector hash remained unchanged. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T06:27:06Z + +AUTOREFRESH_INLINE_FAILURE_20260721 +Attempts 048 and 049 did not appear in immutable history. +Rule: use validated local helper scripts and a simple scheduler entry; never embed the full collector in cron or nested SSH quoting. + +AUTOREFRESH_WRAPPER_PUBLICATION_FAILURE_20260721 +Commands 044-051 did not reach immutable history through homelab-chat-run. +Resolution: hourly snapshots use direct scp plus homelab-runtime-receive publication. + +AUTOREFRESH_052_NOT_PUBLISHED_20260721 +CONTEXT-AUTOREFRESH-ACTIVATE-052 did not appear in immutable history; direct publisher retry requires sanitization before transmission. + +IMMUTABLE_HISTORY_FALSE_NEGATIVE_20260721 +BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative. +Rule: verify the exact COMMAND_ID result, wrapper publication status and commit SHA; never infer absence from a cached directory listing. + +BACKUP_UNSAFE_SOURCE_CAPTURE_071_PARSE_ERROR_20260721 +BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (. +Resolution: avoid nested quote layers; parse targets first and pass file paths as simple SSH arguments. +DETAIL=SECTION_ERROR_REGISTER_END +DETAIL=SECTION_STRUCTURED_ERRORS_INDEX_BEGIN +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "ERRORS-INDEX-004", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "error-ledger", + "started_at_utc": "2026-07-21T07:22:22Z", + "finished_at_utc": "2026-07-21T07:22:22Z", + "reference_register_checked": true, + "reference_sha256": "f6f7749fda946015116cdbbd754c7e7f20356b4a6cf4df330a1b0b8fdf59cbb2", + "error_register_checked": true, + "error_register_sha256": "ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc", + "command_sha256": "a45b24ab6a161508f31ee8a6b954c045e74824b4d12adfef6f298cfac6111016", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "655ec287a9095e799677cb7f87b08d4119cbdf9027d7097fa610da5a022ab30b", + "sanitized_output_sha256": "655ec287a9095e799677cb7f87b08d4119cbdf9027d7097fa610da5a022ab30b", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "{\n \"schema_version\": 1,\n \"status\": \"READY\",\n \"generated_at_utc\": \"2026-07-21T07:22:22.821762Z\",\n \"source\": {\n \"path\": \"/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\",\n \"sha256\": \"ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc\",\n \"line_count\": 934,\n \"sanitized\": true\n },\n \"summary\": {\n \"entry_count\": 154,\n \"rule_count\": 90,\n \"duplicate_entry_ids\": [],\n \"duplicate_entry_signatures\": [\n \"9cd06e064f5a86be9626c4f9a2a819b8d0e71b03309c705fe0e34e1e59f81b73\"\n ],\n \"duplicate_rule_signatures\": [\n \"b936ae30ffb935118585b5a133d9c32643946a3ba5034b85f42bae000518d49a\"\n ]\n },\n \"entries\": [\n {\n \"id\": \"ERR-H-L1-HOMELAB-ASSISTANT-ERROR-REGISTER\",\n \"kind\": \"heading\",\n \"level\": 1,\n \"source_line\": 1,\n \"title\": \"HOMELAB ASSISTANT ERROR REGISTER\",\n \"summary\": \"Назначение: перед каждой следующей командой сверяться с этим файлом.\",\n \"rule_like\": true,\n \"signature\": \"cd4b4d42c26702b03e3e81a620c46dbe3d498f3f2233215024a5c2a49cc1dbfc\"\n },\n {\n \"id\": \"ERR-N-1-L6\",\n \"kind\": \"numbered\",\n \"source_line\": 6,\n \"title\": \"Повторно дал слишком большой интерактивный paste в shell.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"f732778534f22b73e31dead215c9c7078f48a552a5341ae3830eea618885e8a7\"\n },\n {\n \"id\": \"ERR-N-2-L7\",\n \"kind\": \"numbered\",\n \"source_line\": 7,\n \"title\": \"Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"b13c8946e274927bd041960c76208b6f62e22d46f07233388878c1f47afc426a\"\n },\n {\n \"id\": \"ERR-N-3-L8\",\n \"kind\": \"numbered\",\n \"source_line\": 8,\n \"title\": \"Дал генератор справочника прямо в терминал вместо безопасного маленького шага.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"8de70b8d263233557a33a065ee49004ef01e328de0d736eb9e328930119458be\"\n },\n {\n \"id\": \"ERR-N-4-L9\",\n \"kind\": \"numbered\",\n \"source_line\": 9,\n \"title\": \"Нарушил своё же правило: не давать длинные вложенные команды с кавычками.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"d1bf3e42845e6c9d25511814c1da9b21cd8001474f17fd60240194f812f06fe0\"\n },\n {\n \"id\": \"ERR-H-L5-КРИТИЧЕСКИЕ-ОШИБКИ-АССИСТЕНТА\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 5,\n \"title\": \"Критические ошибки ассистента\",\n \"summary\": \"1. Повторно дал слишком большой интерактивный paste в shell. 2. Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт. 3. Дал генератор справочника прямо в терминал вместо безопасного маленького шага. 4. Нарушил своё же правило: не давать длинные вложенные команды с кавычками.\",\n \"rule_like\": false,\n \"signature\": \"0ba065b54f515fe668143ec87d006ea8f27639e6529fe46367ac9acd6f2e28d6\"\n },\n {\n \"id\": \"ERR-H-L11-ЖЁСТКИЕ-ПРАВИЛА-ПЕРЕД-КАЖДОЙ-КОМАНДОЙ\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 11,\n \"title\": \"Жёсткие правила перед каждой командой\",\n \"summary\": \"CHECK-1: команда не должна быть большим paste. CHECK-2: команда не должна содержать большой here-doc. CHECK-3: команда не должна смешивать Markdown, backticks и shell-логику. CHECK-4: команда не должна иметь вложенный ssh с несколькими уровнями кавычек. CHECK-5: команда не должна печатать секреты. CHECK-6: если создаётся файл, сначала маленький безопасный шаг, потом проверка. CHECK-7: если команда длиннее 8 строк, её нельзя давать в интерактивный shell. CHECK-8: для edge-vm использовать debian@[PRIVATE_IP] и sudo.\",\n \"rule_like\": true,\n \"signature\": \"7ac085a9df3e592d1d7d6686ca4980278110a07e296ddc0253cedb075cc08d84\"\n },\n {\n \"id\": \"ERR-H-L23-ТЕКУЩИЕ-ВАЖНЫЕ-ФАКТЫ\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 23,\n \"title\": \"Текущие важные факты\",\n \"summary\": \"Internal network: [PRIVATE_IP]/24. Migration config: migration: secure,network=[PRIVATE_IP]/24. Corosync remains on [PRIVATE_IP]/12/13. VM160 forum-prod is not in Proxmox nightly backup. VM130 edge-vm has secondary disk backup=0 risk. 05_edge_compose_safe.tgz quarantined.\",\n \"rule_like\": false,\n \"signature\": \"ecd7f734df435c41c5f01c9fd7f9371a4012c6019493072741c45c6eaf3ed383\"\n },\n {\n \"id\": \"ERR-N-11-L35\",\n \"kind\": \"numbered\",\n \"source_line\": 35,\n \"title\": \"Ошибка: считать offhost OK после failed rsync.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"aa8a2a5cc3228d73d2feadd69fa2fb1d0f76d020280f69921ac5c00dcfc1f9fd\"\n },\n {\n \"id\": \"ERR-N-12-L40\",\n \"kind\": \"numbered\",\n \"source_line\": 40,\n \"title\": \"Ошибка: широкий secret-поиск по /opt/stacks дал шум.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"da33bb4eaab4ca4827c1e122c4292fa4181a7607db77ebce5d185b867bfe0ac3\"\n },\n {\n \"id\": \"ERR-N-13-L45\",\n \"kind\": \"numbered\",\n \"source_line\": 45,\n \"title\": \"Ошибка: SQL с одинарными кавычками внутри одинарной SSH-команды ломается.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"888cbc312835f6303e74db4c5ed2dcaef1e552e657f83ca6f3f82834eb06c23e\"\n },\n {\n \"id\": \"ERR-N-14-L50\",\n \"kind\": \"numbered\",\n \"source_line\": 50,\n \"title\": \"Ошибка: путать контекст входа и узел выполнения.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"97eec6b76b51ffbc78105ce4e450846c5ff1c85dfbcdfeb3e6c687addd6a1391\"\n },\n {\n \"id\": \"ERR-N-15-L57\",\n \"kind\": \"numbered\",\n \"source_line\": 57,\n \"title\": \"Ошибка: повторно нарушено правило №13 после его добавления.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"95160c6fda4d8bc77fc542236363b182ba66a7314a8032379b033a0feba5e3b2\"\n },\n {\n \"id\": \"ERR-N-16-L63\",\n \"kind\": \"numbered\",\n \"source_line\": 63,\n \"title\": \"Основополагающее правило: перед каждой командой ассистент обязан явно написать: Сверка с файлом ошибок: CHECK пройден.\",\n \"summary\": \"\",\n \"rule_like\": true,\n \"signature\": \"254db5ca3a2611269de7af81ececd43035134e01a8e629c34434913c49eeeb41\"\n },\n {\n \"id\": \"ERR-N-19-L68\",\n \"kind\": \"numbered\",\n \"source_line\": 68,\n \"title\": \"Уточнение формата: не писать отдельную строку сверки перед каждой командой в пакете.\",\n \"summary\": \"\",\n \"rule_like\": true,\n \"signature\": \"a77bda86805176e2dc721961454b31a5bff15fe924bd43496475c5816a8f9650\"\n },\n {\n \"id\": \"ERR-N-20-L73\",\n \"kind\": \"numbered\",\n \"source_line\": 73,\n \"title\": \"Ошибка: широкий XenForo SMTP-поиск по src дал шум исходников.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"bed26484549b8d47c75ee6150c206503c05074cc8a5b25dfde0735596aaf956c\"\n },\n {\n \"id\": \"ERR-N-21-L77\",\n \"kind\": \"numbered\",\n \"source_line\": 77,\n \"title\": \"Ошибка: nested PHP php -r дал Parse error на forum-prod.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"1198d8f0576c5dc8e3883ac7d7e423c8391e946c73bf5ac3cf6063278a68a5af\"\n },\n {\n \"id\": \"ERR-N-22-L82\",\n \"kind\": \"numbered\",\n \"source_line\": 82,\n \"title\": \"Ошибка: самодельный base64 PHP для SMTP auth сломан.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"c629522366fb27522bec6dd9b9b24f532bbc9c3372fbf4d706673bf283bdf498\"\n },\n {\n \"id\": \"ERR-N-23-L87\",\n \"kind\": \"numbered\",\n \"source_line\": 87,\n \"title\": \"Ошибка: MariaDB не поддержала cast('true' as json) в JSON_SET.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"4faccf6aa3103144040fd4efb62dd82e23d9f28ecfba72502293f6e2f060b819\"\n },\n {\n \"id\": \"ERR-N-24-L92\",\n \"kind\": \"numbered\",\n \"source_line\": 92,\n \"title\": \"Ошибка: exit 1 в interactive-check закрыл SSH-сессию.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"c522fbe5bc7aadfc904f1bde7328952b554884941afb3558e60cfe6a4ed378cd\"\n },\n {\n \"id\": \"ERR-N-25-L97\",\n \"kind\": \"numbered\",\n \"source_line\": 97,\n \"title\": \"Ошибка анализа: не надо слепо включать smtpSsl=true при smtpPort=587.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"17417cdf64e208886d887fd449a612090a81aa5a8f47944f17993821ad5e8235\"\n },\n {\n \"id\": \"ERR-N-26-L102\",\n \"kind\": \"numbered\",\n \"source_line\": 102,\n \"title\": \"Ошибка: openssl -crlf вместе с ручным CRLF ломает SMTP AUTH checker.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"3f83176fadcfeb2318a3e3fc75c13e3944d3fef3c0aa7aef40db3736b27e794a\"\n },\n {\n \"id\": \"ERR-N-27-L108\",\n \"kind\": \"numbered\",\n \"source_line\": 108,\n \"title\": \"Ошибка: Python heredoc внутри ssh сломал not_ok диагностику.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"437f761644210f81c8ca3333410a58585c7b74c54a983b9d84159d98c49b4847\"\n },\n {\n \"id\": \"ERR-N-28-L112\",\n \"kind\": \"numbered\",\n \"source_line\": 112,\n \"title\": \"Ошибка: docker compose ls --format json | wc -l дал ложный COMPOSE_PROJECT_COUNT=1.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"752e0913f85275527a448868cce24aa85cf06949ea82e066e4dc0fc2e857d1ad\"\n },\n {\n \"id\": \"ERR-N-29-L116\",\n \"kind\": \"numbered\",\n \"source_line\": 116,\n \"title\": \"Ошибка: monitoring compact status искал неверные имена health-файлов.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"61df0580bfb53db159bb0c3fced23235437e823a877822e0b8aebb7518453d4d\"\n },\n {\n \"id\": \"ERR-N-30-L121\",\n \"kind\": \"numbered\",\n \"source_line\": 121,\n \"title\": \"Ошибка/инцидент: Prometheus API HTTP_CODE=000 после monitoring reference.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"c0dac19354e77153d3f61d53659a8e694c57308bc564e03c833cb68831e88031\"\n },\n {\n \"id\": \"ERR-N-31-L125\",\n \"kind\": \"numbered\",\n \"source_line\": 125,\n \"title\": \"Строгое правило: перед каждой командой сверяться с файлом ошибок и справочником.\",\n \"summary\": \"\",\n \"rule_like\": true,\n \"signature\": \"fa4e49944e2caf8ba4f26a6064bfb465c035e8bc3184244c0314812e81bd6df5\"\n },\n {\n \"id\": \"ERR-N-32-L130\",\n \"kind\": \"numbered\",\n \"source_line\": 130,\n \"title\": \"Ошибка: 153_PROMETHEUS_TARGETS_SETTLED_PROOF содержит SyntaxError из-за вложенного Python внутри SSH.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"13b3234027d04af4194a5352dda2a5bd21ab955751eb161b953637347da8febf\"\n },\n {\n \"id\": \"ERR-N-34-L135\",\n \"kind\": \"numbered\",\n \"source_line\": 135,\n \"title\": \"Ошибка операционного ввода: полный терминальный лог MobaXterm попал в shell.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"9a7930e756982125ac45f47a4adf7dfdb10906bd950db4ddae3659021e01bc64\"\n },\n {\n \"id\": \"ERR-N-33-L140\",\n \"kind\": \"numbered\",\n \"source_line\": 140,\n \"title\": \"Security finding: root authorized_keys на PVE-нодах имел права 777.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"44a7ef2e7d0428388c9b30d12ecfad460caaca457b5d283ebf00ae87c1b2119c\"\n },\n {\n \"id\": \"ERR-N-35-L144\",\n \"kind\": \"numbered\",\n \"source_line\": 144,\n \"title\": \"Ошибка проверки: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED не доказал исправление прав.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"336bfad03f4600ddc0d8996bdbbebeae51cfa0c1e2281ae1a838b8b4d8bc7dfd\"\n },\n {\n \"id\": \"ERR-N-36-L149\",\n \"kind\": \"numbered\",\n \"source_line\": 149,\n \"title\": \"Quality check: Storage block needs integrity and pve03 capacity coverage review.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"a11f29a5da892017620722200a35bdde4c0fe5aa591de4331bfbee0851e38266\"\n },\n {\n \"id\": \"ERR-N-37-L154\",\n \"kind\": \"numbered\",\n \"source_line\": 154,\n \"title\": \"Coverage gap: pve03_staging missing from disk-space health coverage.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"d2fded36e2e512289b08c31b7fc6872e4efbf61c06e2840e5a2dfef9a8e989f8\"\n },\n {\n \"id\": \"ERR-N-38-L158\",\n \"kind\": \"numbered\",\n \"source_line\": 158,\n \"title\": \"Quality check: Service Dependency Map block needs integrity review.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"79cffb917505e112ac9507770da4068e145c75d2278d50d1034bb594b1da5062\"\n },\n {\n \"id\": \"ERR-N-39-L162\",\n \"kind\": \"numbered\",\n \"source_line\": 162,\n \"title\": \"Ошибка проверки: post deep audit BAD_MARKER scan caught prose false positive.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"6dc6e561ff4c49c39a8d2f7fdac4fbfd9d2b6295212185791fb9091f644a2824\"\n },\n {\n \"id\": \"ERR-N-40-L166\",\n \"kind\": \"numbered\",\n \"source_line\": 166,\n \"title\": \"Ошибка/вывод: VM150 Mail cloud single-file upload failed with 413.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"d7e096eac4cbc2b602601af11ce42bb716d28142a278203aefb65699b5fd4b9c\"\n },\n {\n \"id\": \"ERR-N-41-L170\",\n \"kind\": \"numbered\",\n \"source_line\": 170,\n \"title\": \"Ошибка: post-backup-pass-slo-reconciliation checked pve01 router health on edge-vm.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"44ceb289e0963e0e97c6e8019e93120ad638f5648e8596b987e7d413eaeb59aa\"\n },\n {\n \"id\": \"ERR-H-L31-ПРАВИЛО-ДЛЯ-СПРАВОЧНИКА\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 31,\n \"title\": \"Правило для справочника\",\n \"summary\": \"Не генерировать большой справочник через интерактивную вставку. Следующий справочник делать только маленькими append-блоками или через файл, созданный вне интерактивного shell. 11. Ошибка: считать offhost OK после failed rsync. Если rsync/scp упал, нельзя проверять latest offhost-файл без сверки имени. Проверка должна подтвердить именно новый архив, например дату 2026_06_30-00_20_59. Старый OFFHOST_ZSTD_OK не закрывает новый backup. 12. Ошибка: широкий secret-поиск по /opt/stacks дал шум. Не искать Cloudflare/SMTP сразу по /opt/stacks, MkDocs site, backups и HTML-отчётам.\",\n \"rule_like\": true,\n \"signature\": \"2c69d96d9f906cc88ab3158941ba041ca57ddee1e03554fad588eea96041930a\"\n },\n {\n \"id\": \"ERR-H-L174-ASSISTANT-COMMAND-BATCHING-RULE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 174,\n \"title\": \"ASSISTANT_COMMAND_BATCHING_RULE_20260630\",\n \"summary\": \"- Rule: combine related infrastructure checks, proof generation, reference updates and validation scans into one safe batch command when practical. - Do not split work into many small commands when the operations are low-risk, read-only, or naturally part of one evidence block. - Still keep mandatory preflight checks, short bounded commands, no large paste, no here-docs, no interactive shell traps, and no sensitive values in output. - Split into smaller commands only when safety, rollback, quoting risk, or troubleshooting clarity requires it.\",\n \"rule_like\": true,\n \"signature\": \"73ef09d525c334b9874616d5bafdf17d4898a815d35e736e33df0b83790f6539\"\n },\n {\n \"id\": \"ERR-H-L180-HOME-PORTAL-BASE64-APPLY-FAILURE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 180,\n \"title\": \"HOME_PORTAL_BASE64_APPLY_FAILURE_20260630\",\n \"summary\": \"- Prior home-portal apply attempt 312 failed because a large base64 Python payload decoded as invalid UTF-8. - Rule: avoid large opaque base64 script payloads for homelab changes; use transparent bounded shell/perl/python commands instead.\",\n \"rule_like\": true,\n \"signature\": \"4964d3ff556bb377d7c9d0cb4d487afabc0d8ac69c7fb2a5fccb5ff077a8a478\"\n },\n {\n \"id\": \"ERR-H-L184-HOMELAB-COMMAND-SAFETY-HARDENING-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 184,\n \"title\": \"HOMELAB_COMMAND_SAFETY_HARDENING_20260630\",\n \"summary\": \"- Rule: do not use large opaque payloads, generated base64 scripts, or hidden multi-line script blobs for homelab changes. - Use transparent bounded shell/perl/python commands, or create a temporary readable script and validate it before execution. - Any script used for changes must pass a syntax/parse check before it is allowed to modify files or restart services. - Any Traceback, UnicodeDecodeError, SyntaxError, Permission denied, command-not-found, nonzero critical RC, or failed write marker must force REVIEW or FAILED, never OK. - Success requires both runtime health and content-specific post-checks: expected files changed, expected cards exist, duplicates absent, service/container alive, and no new critical log pattern. - HTTP 200 alone is not a success condition for configuration changes. - If an apply step fails, the next command must explicitly record the failed attempt and then run a corrected transparent apply command.\",\n \"rule_like\": true,\n \"signature\": \"463e648fd9e6d671debe81d12b455563460bc669983e083ef5f0f3b4df2f208b\"\n },\n {\n \"id\": \"ERR-H-L193-HOME-PORTAL-API-ERROR-CLOSURE-RULE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 193,\n \"title\": \"HOME_PORTAL_API_ERROR_CLOSURE_RULE_20260630\",\n \"summary\": \"- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200. - If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed. - External informational widgets such as weather/Open-Meteo must not block the service launcher portal.\",\n \"rule_like\": true,\n \"signature\": \"1a011c6be8607fdacb9571baa017241dcb441a0025f985ecce3d0937a4f7e3b5\"\n },\n {\n \"id\": \"ERR-H-L198-HOME-PORTAL-COMPLEX-QUOTING-FAILURE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 198,\n \"title\": \"HOME_PORTAL_COMPLEX_QUOTING_FAILURE_20260630\",\n \"summary\": \"- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('. - Cause: command was too complex and fragile due to nested shell/perl/python quoting. - Rule: do not use deeply nested one-liners for config edits; first run short state checks, then apply small transparent edits.\",\n \"rule_like\": true,\n \"signature\": \"862820180d4dbc46a845af7c9ec8847aeb21cda3fd7634c0d2afd9b3d14d5d8b\"\n },\n {\n \"id\": \"ERR-H-L203-HOME-PORTAL-ACTIVE-CONFIG-SCOPE-RULE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 203,\n \"title\": \"HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RULE_20260630\",\n \"summary\": \"- Rule: Homepage cleanup validation must check active config files only, not logs or backup files. - Active files are services.yaml, bookmarks.yaml, widgets.yaml, settings.yaml and docker.yaml under /opt/stacks/homepage/config. - Matches in /logs/ or *.bak* are historical evidence only and must not make cleanup REVIEW unless the active config still contains the target.\",\n \"rule_like\": true,\n \"signature\": \"6bfd4c200f43d77ecd18a1c61c377b4c0588a028fb0ca6d565b2361d16a74967\"\n },\n {\n \"id\": \"ERR-H-L208-HOME-PORTAL-LINK-VALIDATION-NPMPLUS-DEFAULT-RULE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 208,\n \"title\": \"HOME_PORTAL_LINK_VALIDATION_NPMPLUS_DEFAULT_RULE_20260630\",\n \"summary\": \"- Rule: Homepage link validation must not treat HTTP 200 as success if the body is the NPMPlus default “Congratulations” / host-not-set-up page. - Green siteMonitor dots can be false positives when NPMPlus returns its default vhost page. - A portal card is openable only when DNS, reverse-proxy host mapping, and target content are all valid.\",\n \"rule_like\": true,\n \"signature\": \"90363c457faef01bbe65d3e246073788fb426212371de346152c7776d246413b\"\n },\n {\n \"id\": \"ERR-H-L213-ROUTER-CLI-STDIN-APPLY-FAILURE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 213,\n \"title\": \"ROUTER_CLI_STDIN_APPLY_FAILURE_20260630\",\n \"summary\": \"- Context: attempted Netcraze router ACL apply through SSH stdin/multiline for Homepage Moscow Router monitor fix. - Evidence: 391_MOSCOW_ROUTER_EDGE_VM_WEBADMIN_ACL_APPLY_PROOF.txt. - Failure: STDIN_SHOW_OK=no, APPLY_SKIPPED=stdin_show_failed, CONFIG_HAS_EDGE_5080=no, CONFIG_HAS_EDGE_5083=no. - Rule: do not apply Netcraze config through stdin/multiline SSH; prove CLI input mode first.\",\n \"rule_like\": true,\n \"signature\": \"7a151b61221247f7e5ecc6488283190d82b489caa9309675c08baded25d7fc6f\"\n },\n {\n \"id\": \"ERR-H-L219-ROUTER-CLI-PROBE-STDIN-CONSUMPTION-ERROR-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 219,\n \"title\": \"ROUTER_CLI_PROBE_STDIN_CONSUMPTION_ERROR_20260630\",\n \"summary\": \"- Context: ACL syntax read-only probe loop executed only one command because ssh consumed the loop stdin. - Evidence: 393_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_PROOF.txt showed COMMAND_COUNT=1. - Fix pattern: use ssh -n or redirect SSH stdin away from the command-list loop for all future SSH-in-loop probes.\",\n \"rule_like\": false,\n \"signature\": \"c801838e0f61bbd4f111bc30b21d530e69124b4448859e0e5f28a93cf0f01fec\"\n },\n {\n \"id\": \"ERR-H-L224-ROUTERBACKUP-READONLY-CLI-PERMISSION-LIMIT-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 224,\n \"title\": \"ROUTERBACKUP_READONLY_CLI_PERMISSION_LIMIT_20260630\",\n \"summary\": \"- Context: routerbackup user can run show running-config but cannot execute configuration commands such as access-list. - Evidence: 394_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_FIXED_PROOF.txt. - Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli]. - Rule: do not attempt ACL/config changes as routerbackup; use admin/operator router access or router Web UI for write changes.\",\n \"rule_like\": true,\n \"signature\": \"1d8075c128dbd4b8f34929055e7c4b7c8cfda8f44f0b7874f04afbcc745d5d82\"\n },\n {\n \"id\": \"ERR-H-L230-PROOF-SUMMARY-EXTRACTION-BLANK-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 230,\n \"title\": \"PROOF_SUMMARY_EXTRACTION_BLANK_20260630\",\n \"summary\": \"- Context: proof 395 tried to read COMMAND_COUNT/ZERO_RC_COUNT from proof files 393/394, but those summary fields were printed to console and were not persisted inside the proof files. - Evidence: 395_ROUTER_CLI_ERROR_REGISTER_UPDATE_PROOF.txt has blank P393_COMMAND_COUNT/P394_COMMAND_COUNT/P394_ZERO_RC_COUNT fields. - Rule: when creating follow-up proof summaries, derive counts from persisted proof body lines such as CMD_INDEX and CMD_RC, or write summary fields into the proof file itself.\",\n \"rule_like\": false,\n \"signature\": \"1fd6a207e921a706e6d8d3e80984d0436e67192bb56d03c91d20f6b33eba15cd\"\n },\n {\n \"id\": \"ERR-H-L235-ROUTER-ACL-UI-RULE-DELETE-AND-RESTORE-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 235,\n \"title\": \"ROUTER_ACL_UI_RULE_DELETE_AND_RESTORE_20260701\",\n \"summary\": \"- Context: while moving Moscow Router monitor rules to _WEBADMIN_Bridge1, two _WEBADMIN_Bridge0 Nextcloud Talk TURN rules were accidentally removed. - Deleted/restored rules: Home [PRIVATE_IP]/24 to Nextcloud [PRIVATE_IP] TCP/UDP 3478. - Verification proof: 399_ROUTER_ACL_RESTORE_AFTER_UI_PROOF.txt. - Rule: after manual router Web UI ACL edits, compare _WEBADMIN_Bridge0, _WEBADMIN_Bridge1 and _WEBADMIN_GigabitEthernet1 before continuing.\",\n \"rule_like\": false,\n \"signature\": \"973a47bd8e8f388786157b6d50b53de9fc96b31e582a9645f3cf4189b74ac4fe\"\n },\n {\n \"id\": \"ERR-H-L241-ROUTER-MONITOR-NODE-LOOKUP-PROBE-BUG-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 241,\n \"title\": \"ROUTER_MONITOR_NODE_LOOKUP_PROBE_BUG_20260701\",\n \"summary\": \"- Context: proof 400 tested Moscow router HTTPS monitor from Homepage container with custom Node lookup callback. - Failure: NODE_HTTPS_RESOLVE_ERROR=ERR_INVALID_IP_ADDRESS and NODE_HTTPS_RESOLVE_INSECURE_ERROR=ERR_INVALID_IP_ADDRESS. - Impact: Node HTTPS resolve subtest is invalid; curl --resolve results remain valid. - Rule: do not rely on custom Node lookup callback until separately tested; prefer curl --resolve or simple direct Node HTTP checks.\",\n \"rule_like\": true,\n \"signature\": \"97151454d5ffda69d45b802ed15dc82057b02632e141b3729fa8ad47a959c556\"\n },\n {\n \"id\": \"ERR-H-L247-ROUTER-HTTP-SERVICE-FORBIDDEN-FROM-EDGE-VM-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 247,\n \"title\": \"ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701\",\n \"summary\": \"- Context: Moscow Router Homepage monitor after ACL fix. - Evidence: proofs 399, 400, 401. - Observed: edge-vm TCP to [PRIVATE_IP]:5080/5083 is OK, but router HTTP/HTTPS returns 403 for all tested paths. - Impact: Homepage siteMonitor cannot be made green via direct router HTTP URL until router web service allows the Bridge1/Proxmox/protected segment or an explicit safe monitor endpoint is used. - Rule: do not keep changing ACL for this symptom; ACL is already correct. Fix service access scope or use a dedicated monitor endpoint.\",\n \"rule_like\": true,\n \"signature\": \"b0de3053a46b06841cbaf6ea5945be49ad8d451c7c8453d899d01a9f520cc384\"\n },\n {\n \"id\": \"ERR-H-L254-PY-COMPILE-PYC-PERMISSION-ERROR-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 254,\n \"title\": \"PY_COMPILE_PYC_PERMISSION_ERROR_20260701\",\n \"summary\": \"- Context: installing edge-vm Moscow router health endpoint. - Mistake: assistant used `python3 -m py_compile` on a script installed under /usr/local/sbin as root. - Actual impact: non-root syntax check attempted to create /usr/local/sbin/__pycache__ and failed with Permission denied. - Recovery: validate syntax with `ast.parse` instead of py_compile, because it does not write .pyc files. - Rule: do not use py_compile against root-owned system paths from an unprivileged user.\",\n \"rule_like\": true,\n \"signature\": \"683a661513a8d46a89bf6492c8f12564f6f9fe262ce16b0f6487f63c0f2666f0\"\n },\n {\n \"id\": \"ERR-H-L261-ROUTER-HOME-IP-DOCKER-ROUTE-CONFLICT-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 261,\n \"title\": \"ROUTER_HOME_IP_DOCKER_ROUTE_CONFLICT_20260701\",\n \"summary\": \"- Context: Moscow Router Homepage siteMonitor attempt using http://[PRIVATE_IP]:5080. - Evidence: proof 405 showed route to [PRIVATE_IP] via Docker bridge and Homepage container ECONNRESET. - Impact: do not use [PRIVATE_IP] directly as Homepage siteMonitor from edge-vm. - Mitigation: use dedicated edge-vm health endpoint that checks router TCP [PRIVATE_IP]:5080 and returns HTTP 200/503 for Homepage.\",\n \"rule_like\": true,\n \"signature\": \"5daeb3cf0b7d3bf64ddf6ca87e958f73d0715672617706dee233fd739fb08c2c\"\n },\n {\n \"id\": \"ERR-H-L267-HOMEPAGE-ROUTER-MOSCOW-YAML-TITLE-SHAPE-MISMATCH-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 267,\n \"title\": \"HOMEPAGE_ROUTER_MOSCOW_YAML_TITLE_SHAPE_MISMATCH_20260701\",\n \"summary\": \"- Context: applying Moscow Router Homepage siteMonitor health endpoint. - Mistake: assistant apply script matched only property-style cards with `title`, but active Homepage YAML can use service-name-key style like `- Service Name:`. - Evidence: proof 407 returned YAML_CHANGE_COUNT=0 and APPLY_SKIPPED=unexpected_match_count while health endpoint proof 406 was OK. - Recovery: use YAML-aware updater that handles both property-style and service-name-key style cards.\",\n \"rule_like\": false,\n \"signature\": \"d7ed6a0ecfb19584319e97626e76661541c675e9125e4bf4ff0a407c9718cdd0\"\n },\n {\n \"id\": \"ERR-H-L273-HOMEPAGE-ROUTER-MOSCOW-APPLY-SCRIPT-SYNTAX-ERROR-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 273,\n \"title\": \"HOMEPAGE_ROUTER_MOSCOW_APPLY_SCRIPT_SYNTAX_ERROR_20260701\",\n \"summary\": \"- Context: applying Moscow Router Homepage health endpoint. - Mistake: assistant generated Python script with invalid f-string escaping. - Evidence: proof 408 showed SyntaxError in /tmp/homepage-router-moscow-apply-fixed.py. - Actual impact: YAML was not changed, so Homepage green dot could not appear. - Recovery: stop applying blind updater; first discover exact active YAML shape, then apply minimal YAML-aware edit. - Rule: validate generated Python syntax locally before scp/remote execution; avoid f-strings in generated maintenance scripts.\",\n \"rule_like\": true,\n \"signature\": \"057b7e036316f6f240cac06b9f46e98d8399df689f82b903cc9ef630afe09f94\"\n },\n {\n \"id\": \"ERR-H-L281-FORUM-PROD-VM160-FIRST-BOOT-SSH255-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 281,\n \"title\": \"FORUM_PROD_VM160_FIRST_BOOT_SSH255_20260701\",\n \"summary\": \"- Context: clean rebuilt VM160 first boot. - Symptom: PING_OK=yes and TCP22_OK=yes, but SSH_RC=255. - Rule: do not proceed with OS baseline until SSH failure is diagnosed; likely old known_hosts key or cloud-init/root-key issue.\",\n \"rule_like\": true,\n \"signature\": \"5e0359dc6b12e39f533e9060876f5c5f830b43cf2dbbe0441d6876fee5f410a3\"\n },\n {\n \"id\": \"ERR-H-L286-FORUM-PROD-VM160-NESTED-HOSTNAME-PROOF-QUOTING-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 286,\n \"title\": \"FORUM_PROD_VM160_NESTED_HOSTNAME_PROOF_QUOTING_20260701\",\n \"summary\": \"- Context: VM160 first SSH proof after rebuild. - Issue: command substitution $(hostname) inside nested ssh was expanded on pve02 before entering VM160. - Impact: VM_HOSTNAME=pve02 in proof 426 is not valid VM identity evidence. - Rule: for nested SSH identity checks, run literal hostname commands without local command substitution.\",\n \"rule_like\": false,\n \"signature\": \"c7174e4703945abd82004fea3fa9da0ff3efa1f2a89e6d2315cacf9c27b6d761\"\n },\n {\n \"id\": \"ERR-H-L292-FORUM-PROD-VM160-SWAP-PROOF-QUOTING-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 292,\n \"title\": \"FORUM_PROD_VM160_SWAP_PROOF_QUOTING_20260701\",\n \"summary\": \"- Context: VM160 swapfile proof 429. - Issue: nested command substitution expanded on pve02, producing invalid SWAPFILE/FSTAB proof lines. - Impact: proof 429 is not valid closure evidence even though swap was active. - Rule: avoid nested $(...) in VM SSH proofs; use literal remote commands and clean proof.\",\n \"rule_like\": true,\n \"signature\": \"3dc2ea2b0a54adb8886979de4e01cb5f23704446f5b2198af9bdacc951359e4c\"\n },\n {\n \"id\": \"ERR-H-L298-FORUM-UPLOAD-ARCHIVES-VM-VERIFY-QUOTING-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 298,\n \"title\": \"FORUM_UPLOAD_ARCHIVES_VM_VERIFY_QUOTING_20260701\",\n \"summary\": \"- Context: proof 446 copy/check archives inside VM160. - Issue: nested SSH quoting expanded shell variables incorrectly, producing gzip checks against empty .gz and blank TAR_TOP lines. - Impact: SCP_TO_VM_RC=0 and file size list are valid, but GZIP/TAR proof lines in 446 are invalid. - Rule: verify VM archive integrity locally inside forum-prod, not through nested quoted loops.\",\n \"rule_like\": false,\n \"signature\": \"0e4a3a2f7c179f46c235e1ffa176b57485c88b4f7e7314bb89b6acb6683fe448\"\n },\n {\n \"id\": \"ERR-H-L304-FORUM-PROD-BULK-IMPORT-PHP85-EMPTY-FRONTEND-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 304,\n \"title\": \"FORUM_PROD_BULK_IMPORT_PHP85_EMPTY_FRONTEND_20260701\",\n \"summary\": \"- Context: rebuilt VM160, five XenForo forums imported in bulk on PHP 8.5.7. - Symptom: all five forums returned HTTP 200 with empty body; XenForo logged Template public:PAGE_CONTAINER/forum_list is unknown. - Additional failed approach: proof 480 ran xf-dev:rebuild-caches on CodeVipe and returned RC=1, not a valid fix. - Decision: stop bulk repair, treat current VM160 as lab, find a working CodeVipe recipe first, then destroy/rebuild cleanly for all five forums. - Rule: no more mass rebuild/repair commands across all forums until single-forum CodeVipe recipe is proven.\",\n \"rule_like\": false,\n \"signature\": \"ea3c74648eece7c2a94559d8f7385ea4a126b399e3a161c5113e69a7b8d05f93\"\n },\n {\n \"id\": \"ERR-H-L311-XENFORO-REQUIREMENTS-DIRECT-DOWNLOAD-403-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 311,\n \"title\": \"XENFORO_REQUIREMENTS_DIRECT_DOWNLOAD_403_20260701\",\n \"summary\": \"- Context: Tried to download XenForo requirements ZIP directly from xenforo.com inside forum-prod. - Issue: curl returned HTTP 403; unzip then failed because the ZIP was not downloaded. - Evidence: proof 491 showed DOWNLOAD_RC=22, ZIP_LIST_RC=9, UNZIP_RC=9. - Impact: proof 491 is not a valid server compatibility test. - Rule: use operator-uploaded xenforo23-requirements-test.zip from /root/forum-upload instead of relying on direct curl download.\",\n \"rule_like\": false,\n \"signature\": \"13164e6750f679553f0bb7cb13f4b895f345dc3f76c978349dfd30b93294206b\"\n },\n {\n \"id\": \"ERR-H-L318-PVE01-UNZIP-MISSING-FOR-XF-REQUIREMENTS-VERIFY-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 318,\n \"title\": \"PVE01_UNZIP_MISSING_FOR_XF_REQUIREMENTS_VERIFY_20260701\",\n \"summary\": \"- Context: proof 492 tried to inspect uploaded xenforo23-requirements-test.zip on pve01. - Issue: pve01 does not have unzip installed, so ZIP_TEST_RC=127 was not a ZIP integrity result. - Impact: proof 492 confirmed file presence only, not archive validity. - Rule: verify ZIP with Python zipfile or inside forum-prod rather than installing unzip on Proxmox just for inspection.\",\n \"rule_like\": false,\n \"signature\": \"341e814b6c0aab80d0bde631353118c8c04593d918a2ed7b60551d5bc3043148\"\n },\n {\n \"id\": \"ERR-H-L324-FRESH5-DEPLOY-SUCCESS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 324,\n \"title\": \"FRESH5_DEPLOY_SUCCESS_20260701\",\n \"summary\": \"- Context: VM160 was destroyed/recreated, then five XenForo forums were deployed from fresh ZIP backups. - Result: proof 513 confirms all five forums locally healthy. - Note: proof 511 initially failed dsmods path detection; proof 512 fixed dsmods by locating public_html via src/XF.php marker. - Rule: future restores should preserve internal_data/code_cache or rebuild compiled template cache before smoke testing.\",\n \"rule_like\": false,\n \"signature\": \"f46e9779cb65eebc42015f7042a4f55f0a8aa1e59e647eb29d728e48ed6069b3\"\n },\n {\n \"id\": \"ERR-H-L330-NPMPLUS-SQLITE-PASTE-FAILURE-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 330,\n \"title\": \"NPMPLUS_SQLITE_PASTE_FAILURE_20260701\",\n \"summary\": \"- Context: NPMplus SQLite inspection command was too complex and a Python fragment was pasted into bash. - Issue: shell entered multiline prompt and produced syntax errors. - Impact: do not trust that SQLite inspection attempt. - Rule: avoid nested SSH heredoc/Python for this task; use simpler commands, generated nginx configs, or uploaded/local script files.\",\n \"rule_like\": true,\n \"signature\": \"d1f2dda1b36bc12803264780b71545d7d9b00c4bdb88ef2fa85405e9d27cdf6f\"\n },\n {\n \"id\": \"ERR-H-L336-NPMPLUS-PUBLISH-SCRIPT-WRONG-ENV-SOURCE-525-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 336,\n \"title\": \"NPMPLUS_PUBLISH_SCRIPT_WRONG_ENV_SOURCE_525_20260701\",\n \"summary\": \"- Context: proof 525 tried to create forum proxy hosts in NPMplus. - Issue: script sourced /opt/npmplus/.env, but INITIAL_ADMIN_EMAIL/PASSWORD were not present there; they are in the npmplus container env. - Impact: no forum proxy hosts were created by proof 525. - Rule: read NPMplus API login values from docker inspect env internally, never print them.\",\n \"rule_like\": true,\n \"signature\": \"58aa4eeee2841a42e270226a408177af9a099631e2a47857c825707b2ac523ff\"\n },\n {\n \"id\": \"ERR-H-L342-NPMPLUS-API-LOGIN-MISSED-HTTPS81-AND-JSON-ESCAPE-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 342,\n \"title\": \"NPMPLUS_API_LOGIN_MISSED_HTTPS81_AND_JSON_ESCAPE_20260701\",\n \"summary\": \"- Context: NPMplus API login attempts in proofs 526/527 failed. - Issue: commands missed the likely HTTPS admin endpoint on port 81 and built JSON by string interpolation instead of jq escaping. - Impact: no proxy hosts were created by 526/527. - Rule: for NPMplus API use jq-generated JSON and test https://127.0.0.1:81/api/tokens with -k before publish.\",\n \"rule_like\": false,\n \"signature\": \"141ce6d2777003569c531b5308e56cffaea92dc113fdb0c9e9ebe45815805101\"\n },\n {\n \"id\": \"ERR-H-L348-NPMPLUS-API-AUTH-UNAVAILABLE-MANUAL-ROUTES-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 348,\n \"title\": \"NPMPLUS_API_AUTH_UNAVAILABLE_MANUAL_ROUTES_20260701\",\n \"summary\": \"- Context: NPMplus API login attempts failed even though the forum VM and edge-to-forum connectivity are healthy. - Issue: API credentials from container initial env are not accepted by current NPMplus API. - Impact: do not use NPMplus API for this publish path. - Rule: publish forum routes through backed-up manual nginx proxy_host configs, then issue proper certificates after DNS points to edge.\",\n \"rule_like\": true,\n \"signature\": \"d5a4f4c6312277e9e2c1487b38c4bf554f4b9618fad9e2c5328fe8324e26a555\"\n },\n {\n \"id\": \"ERR-H-L354-EDGE-CERTBOT-TOKEN-FILE-PERMISSION-535-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 354,\n \"title\": \"EDGE_CERTBOT_TOKEN_FILE_PERMISSION_535_20260701\",\n \"summary\": \"- Context: proof 535 attempted DNS-01 certificate issue for five forum domains. - Issue: /tmp/forum_cf_token.env was root-owned mode 600, so debian user could not source it directly. - Impact: certbot ran without token and all certificate attempts failed; DNS cutover must not proceed from proof 535. - Rule: read Cloudflare token via sudo without printing it, then create temporary certbot credentials file and remove it after use.\",\n \"rule_like\": true,\n \"signature\": \"4566b4ef97181c31efcb32eefd3ed52633883fc929e98d4607a6c48671434582\"\n },\n {\n \"id\": \"ERR-H-L360-EDGE-CERTBOT-FAILED-DNS-CUTOVER-ABORTED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 360,\n \"title\": \"EDGE_CERTBOT_FAILED_DNS_CUTOVER_ABORTED_20260701\",\n \"summary\": \"- Context: edge manual routes are working locally, but certbot DNS-01 certificate issue failed in proof 538. - Issue: all five certbot attempts returned RC=1; proof 539 correctly aborted DNS cutover because edge cert precheck was not ready. - Impact: Cloudflare DNS records were not changed by proof 539; public DNS still points to old IP 87.236.18.45. - Rule: inspect certbot logs and Cloudflare token zone permissions before retrying certificate issuance or DNS cutover.\",\n \"rule_like\": false,\n \"signature\": \"7fc8883c0f715d038cc77e38a9088d6cb18290d2456232e9e0a13b1ba094d256\"\n },\n {\n \"id\": \"ERR-H-L366-CLOUDFLARE-TOKEN-NO-ZONE-ACCESS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 366,\n \"title\": \"CLOUDFLARE_TOKEN_NO_ZONE_ACCESS_20260701\",\n \"summary\": \"- Context: Cloudflare token discovered on edge verified as a token but could not access forum zones. - Evidence: proof 542 showed all five zones missing and DNS record create probes failed. - Impact: certbot DNS-01 failed and DNS cutover was correctly aborted; public DNS still points to old hosting. - Rule: install a new Cloudflare token with Zone Read and DNS Edit for the five forum zones before retrying certs/DNS.\",\n \"rule_like\": false,\n \"signature\": \"3ba4fdbb1dcc5d0beeb215153e4471c19e29f4080ff3d061e77154f44de9a898\"\n },\n {\n \"id\": \"ERR-H-L372-FORUM-PUBLICATION-FINAL-SUCCESS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 372,\n \"title\": \"FORUM_PUBLICATION_FINAL_SUCCESS_20260701\",\n \"summary\": \"- Context: five XenForo forums were rebuilt on VM160, routed through edge NPMplus, issued Let’s Encrypt certs using a corrected Cloudflare token, and cut over in DNS. - Result: final public proof 546 passed. - Important corrections: initial NPMplus API route failed; manual Nginx routes were used. Initial Cloudflare token had no zone access; corrected token passed zone/DNS probe.\",\n \"rule_like\": false,\n \"signature\": \"bec7980ce2230b8ecf69cfc694cc56bbd3dedc45da55ed2df567e8f41b228638\"\n },\n {\n \"id\": \"ERR-H-L377-FINAL-SNAPSHOT-NAME-TOO-LONG-556-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 377,\n \"title\": \"FINAL_SNAPSHOT_NAME_TOO_LONG_556_20260701\",\n \"summary\": \"- Context: proof 556 final health gate passed for all five public forums. - Issue: Proxmox snapshot failed because generated snapshot name exceeded 40 characters. - Evidence: qm snapshot returned snapname value may only be 40 characters long. - Impact: forum health was OK, but proof 556 snapshot step was not completed. - Fix: rerun snapshot with short name.\",\n \"rule_like\": false,\n \"signature\": \"c3d65c37278c1e067af0f6dc1d7eb36fa6e75b03113162e66adf06c483bedc76\"\n },\n {\n \"id\": \"ERR-H-L384-CLOUDFLARE-DNS-AUDIT-TXT-SUBSTRING-GAP-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 384,\n \"title\": \"CLOUDFLARE_DNS_AUDIT_TXT_SUBSTRING_GAP_20260701\",\n \"summary\": \"- Context: prior Cloudflare DNS audit checked web A/AAAA/CNAME state for root/www records. - Issue: it did not deeply inspect TXT/SPF contents for old provider IP substrings. - Evidence: user screenshot showed SPF TXT containing old 87.236.18.* value. - Impact: web routing is OK, but mail-related DNS may still contain stale provider data. - Rule: perform full DNS record audit including TXT/SPF/DKIM/DMARC/MX/CDN and substring search for old IPs before DNS cleanup.\",\n \"rule_like\": false,\n \"signature\": \"260c1f016ae9bf9efb5888bb54809e742681b512223781efac0a4cc8d8cdf2c7\"\n },\n {\n \"id\": \"ERR-H-L391-CLOUDFLARE-DNS-CLEANUP-DRYRUN-QUOTE-563-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 391,\n \"title\": \"CLOUDFLARE_DNS_CLEANUP_DRYRUN_QUOTE_563_20260701\",\n \"summary\": \"- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt. - Impact: proof 563 is invalid and no DNS cleanup was performed by it. - Rule: use copied scripts for Cloudflare DNS mutation instead of fragile inline nested quotes.\",\n \"rule_like\": false,\n \"signature\": \"3837def1e15ccdc91af766587f44bd7e98885fb25b88ca3d62e65d13698f62d1\"\n },\n {\n \"id\": \"ERR-H-L396-SPF-DUPLICATE-AFTER-565-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 396,\n \"title\": \"SPF_DUPLICATE_AFTER_565_20260701\",\n \"summary\": \"- Context: SPF cleanup command 565 attempted to replace stale SPF records. - Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained. - Impact: domains temporarily had duplicate SPF records, which is invalid for mail validation. - Fix: delete all SPF TXT records for the five forum zones, then create exactly one v=spf1 -all TXT per zone.\",\n \"rule_like\": false,\n \"signature\": \"4a4b1cba0ac945aef40797821d7a9a774728f30fb5e56c34599575e1b5b40ba8\"\n },\n {\n \"id\": \"ERR-H-L402-MSMTP-SECRET-SOURCE-PARSE-ERROR-576-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 402,\n \"title\": \"MSMTP_SECRET_SOURCE_PARSE_ERROR_576_20260701\",\n \"summary\": \"- Context: proof 576 installed msmtp but sendmail auth test failed. - Issue: [SENSITIVE_PATH] was sourced as shell, but SMTP password contained shell-special characters; source failed and auth used an invalid/truncated secret path. - Impact: msmtp package installed, but mail sending was not proven working. - Rule: never source SMTP secret files containing arbitrary passwords; store password base64 and use msmtp passwordeval helper accessible to www-data.\",\n \"rule_like\": true,\n \"signature\": \"6288f1fcabcdeda1884be858bea052ff9dbb35a65054f5cff6315a350dd7ef12\"\n },\n {\n \"id\": \"ERR-H-L408-FORUM-MSMTP-MAIL-TRANSPORT-STILL-FAILING-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 408,\n \"title\": \"FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701\",\n \"summary\": \"- Context: attempted to fix msmtp config with passwordeval helper. - Issue: one or more sendmail/PHP mail tests still failed. - Impact: XenForo mail sending is not yet proven. - Next step: verify SMTP credential/password and provider policy for noreply@pvepro.ru.\",\n \"rule_like\": false,\n \"signature\": \"31d614a3bb69ec05a33a5be14388347a438e13f1e28a7ec58207273159e00eae\"\n },\n {\n \"id\": \"ERR-H-L414-SMTP-PASSWORD-EXPOSED-ROTATE-REQUIRED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 414,\n \"title\": \"SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701\",\n \"summary\": \"- Context: SMTP password was exposed in terminal output during failed msmtp setup. - Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line. - Impact: treat that SMTP password as compromised. - Required action: rotate the mailbox/app password in the mail provider panel before any further SMTP testing. - Rule: never store arbitrary passwords as shell-sourceable KEY=value files; use non-printing secret storage only.\",\n \"rule_like\": true,\n \"signature\": \"88ae0aeb13d948e6f2cb2d83dc2a9464e4de1805cdb4271cf78f3d315f7095c9\"\n },\n {\n \"id\": \"ERR-H-L421-NETBIRD-MAILCOW-REPORTED-DOWN-AFTER-SMTP-INCIDENT-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 421,\n \"title\": \"NETBIRD_MAILCOW_REPORTED_DOWN_AFTER_SMTP_INCIDENT_20260701\",\n \"summary\": \"- Context: user reported NetBird and Mailcow became unavailable after forum mail-transport work. - Impact: treat as active incident until service reachability and container/VM state are proven. - Rule: do not change DNS, firewall, proxy, mail, NetBird, or Mailcow configs before read-only triage proof.\",\n \"rule_like\": true,\n \"signature\": \"b0eef26c0503e63d8a8c5a294ea1e124ab5e6ac8936d2ccaf23cd9b74fc5ebf3\"\n },\n {\n \"id\": \"ERR-H-L426-FORUM-SMTP-FAILED-CONFIG-DISABLED-AFTER-MAILCOW-BAN-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 426,\n \"title\": \"FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701\",\n \"summary\": \"- Context: forum-prod SMTP/msmtp test used invalid or compromised credentials and triggered Mailcow netfilter warnings/ban for 95.84.154.183. - Impact: forum mail transport is not proven and must remain disabled until SMTP password is rotated and tested once carefully. - Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth. - Rule: do not test SMTP auth again until a fresh rotated mailbox/app password is available.\",\n \"rule_like\": true,\n \"signature\": \"70dc70d8c15e3f30611c45123bf2c3b27c86bf66d703df3e4232057d380801d6\"\n },\n {\n \"id\": \"ERR-H-L432-FORUM-OLD-CODEVIPE-CLOUD-BACKUP-STALE-AFTER-FRESH5-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 432,\n \"title\": \"FORUM_OLD_CODEVIPE_CLOUD_BACKUP_STALE_AFTER_FRESH5_20260701\",\n \"summary\": \"- Context: pve02 codevipe-cloud-backup.timer existed before fresh five-forum rebuild. - Issue: old script points to /root/backups/codevipe and old remote path app/codevipe/forum-prod/local. - Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory contains old codevipe-public/codevipe.sql files, not current five-forum archive. - Impact: old timer must not be treated as valid current backup for all five forums. - Fix: create a new forum-prod fresh5 cloud backup using existing pve02 Mail-cloud rclone crypt remotes.\",\n \"rule_like\": true,\n \"signature\": \"3baaa9a93ec07abd251041838b9602a52b143ef8e07a03dde6928db27eb2b44f\"\n },\n {\n \"id\": \"ERR-H-L439-FORUM-RESTORE-DRILL-610-STATUS-FLAG-BUG-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 439,\n \"title\": \"FORUM_RESTORE_DRILL_610_STATUS_FLAG_BUG_20260701\",\n \"summary\": \"- Context: restore drill proof 610 downloaded Mail.ru Cloud backup, verified split part SHA256, reconstructed archive, matched full archive SHA256, and confirmed all five forum DB/file archives in tar. - Issue: final RESTORE_DRILL_OK was set to no because the script tested an unassigned shell variable RCLONE_REMOTE_PVE02_MAIL_01_RC. - Impact: proof 610 final status flag is invalid, but its detailed integrity checks are valid. - Fix: rerun corrected restore validation against the downloaded cloud backup artifacts and produce proof 612.\",\n \"rule_like\": false,\n \"signature\": \"fd5f6f353d76ced7d2deb5e924a183c516c80e7329d8b9d5c4be0627b5b92d41\"\n },\n {\n \"id\": \"ERR-H-L445-FORUM-XENFORO-MAIL-SMOKE-TEST-FAILED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 445,\n \"title\": \"FORUM_XENFORO_MAIL_SMOKE_TEST_FAILED_20260701\",\n \"summary\": \"- XenForo-level mail smoke test did not return success for all five forums. - Check proof 623 and msmtp log before retrying.\",\n \"rule_like\": false,\n \"signature\": \"c48950d0c079bf801e38017835444fdd04a5354397908bbd864f625e7a917b3d\"\n },\n {\n \"id\": \"ERR-H-L449-XENFORO-MAIL-SMOKE-623-INVALID-ROOT-PATH-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 449,\n \"title\": \"XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701\",\n \"summary\": \"- Context: custom XenForo mail smoke test 623 failed with Could not open input file /root/scripts/623_xenforo_mail_smoke.php. - Cause: the script was run as www-data from /root/scripts; /root is not traversable by www-data. - Impact: proof 623 is invalid and should not be used to judge mail delivery. - Superseding evidence: user ran XenForo built-in outgoing email test; message was delivered to Yandex but placed in Spam. - Next step: investigate deliverability/authentication headers and pvepro.ru SPF/DKIM/DMARC/PTR.\",\n \"rule_like\": false,\n \"signature\": \"ad9141334766e53e58aef9b0fb014ebfbfe556d3d36615b5f4754e2b29cf36fd\"\n },\n {\n \"id\": \"ERR-H-L456-FORUM-NEXT-CHAT-KNOWN-ERRORS-AND-CLOSED-INCIDENTS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 456,\n \"title\": \"FORUM_NEXT_CHAT_KNOWN_ERRORS_AND_CLOSED_INCIDENTS_20260701\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"d4866c37a32090d3456bbb2b44824481a232211fdef57833b7f7bec57c065690\"\n },\n {\n \"id\": \"ERR-H-L458-CLOSED-CLASSIFIED-INCIDENTS\",\n \"kind\": \"heading\",\n \"level\": 3,\n \"source_line\": 458,\n \"title\": \"Closed / classified incidents\",\n \"summary\": \"- SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701: - A previous bad command sourced a raw SMTP secret and printed it. - Treat old password as compromised. - Later persistent SMTP was rebuilt using safe files and verified. - Never print or package secrets. - XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701: - Custom mail proof 623 failed with \\\"Could not open input file\\\". - Cause: PHP was run as www-data from /root/scripts, but /root is not traversable by www-data.\",\n \"rule_like\": true,\n \"signature\": \"eafc804db9ff3f7b3b68c10318b53db82160fe7951c6e68c4301703c2b7abe41\"\n },\n {\n \"id\": \"ERR-H-L491-CURRENT-NON-BLOCKING-ITEMS\",\n \"kind\": \"heading\",\n \"level\": 3,\n \"source_line\": 491,\n \"title\": \"Current non-blocking items\",\n \"summary\": \"- Yandex placed built-in XenForo test mail in Spam despite SPF pass and DKIM pass. - Classification: deliverability/reputation/content filtering, not server failure. - Recommended manual action: click “Не спам!” and add/check pvepro.ru in Yandex Postmaster.\",\n \"rule_like\": false,\n \"signature\": \"bc80a4393b6c0958e5753d3114c2ff3e0554b960a6fed7b7829625f02a9d595a\"\n },\n {\n \"id\": \"ERR-H-L496-SAFETY-RULES-FOR-NEXT-CHAT\",\n \"kind\": \"heading\",\n \"level\": 3,\n \"source_line\": 496,\n \"title\": \"Safety rules for next chat\",\n \"summary\": \"- Do not print secrets. - Do not download or upload: - [SENSITIVE_PATH] - /etc/msmtprc - /etc/msmtp/* - rclone configs - Cloudflare tokens - DB dumps\",\n \"rule_like\": true,\n \"signature\": \"e95262ee083c9c3d1b8587e182598186f1be23e5db9229b5aece7ea336fa6ca5\"\n },\n {\n \"id\": \"ERR-H-L509-PARKED-DOMAINS-STAGE4-DNS01-PREFLIGHT-FAILED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 509,\n \"title\": \"PARKED_DOMAINS_STAGE4_DNS01_PREFLIGHT_FAILED_20260701\",\n \"summary\": \"- Context: parked-domain public apply proof 634. - Issue: DNS-01 based apply was attempted before a clear CAN_APPLY=yes preflight result was visible in terminal output. - Observed: stage4 stopped safely with preflight_failed; CERTBOT_PRESENT=no, CERTBOT_DNS_CLOUDFLARE_PLUGIN=no, and CF_ZONE_ACCESS=no for newfi.ru, hapusya.ru and kingofwolk.ru. - Impact: no parked-domain NPMplus route/certificate changes were applied by stage4; public HTTPS validation still failed certificate verification. - Rule: do not use Cloudflare DNS-01 for these parked domains unless a token with explicit zone access is installed; because DNS already points to edge, prefer a copied-script HTTP-01 flow with temporary ACME challenge routes and content-specific validation.\",\n \"rule_like\": true,\n \"signature\": \"9a52142ed4109dad52b9391a4a479df84e258ff703bfd8904e4ec5947589cd35\"\n },\n {\n \"id\": \"ERR-H-L516-PARKED-DOMAINS-STAGE5-BASH-LOCAL-SETU-BUG-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 516,\n \"title\": \"PARKED_DOMAINS_STAGE5_BASH_LOCAL_SETU_BUG_20260701\",\n \"summary\": \"- Context: parked-domain HTTP-01 apply proof 635. - Issue: edge script used Bash `local id=\\\"$1\\\" ... conf=\\\"$WORK/.../$id.conf\\\"` and `local host=\\\"$1\\\" ... tmp=\\\"$WORK/.../$host.html\\\"` under `set -u`; dependent variables are not safe inside the same local assignment command. - Observed: `id: unbound variable`, `source can not be empty`, then `host: unbound variable`; certbot did not run, final HTTPS routes were not installed. - Impact: local parked page remained OK; public HTTPS remained not closed. - Rule: split dependent Bash local assignments into separate statements, syntax-check copied scripts before execution, and keep content-specific validation for parked routes.\",\n \"rule_like\": false,\n \"signature\": \"c5a97f3f8380803056dcbb2abd47a72f37609e59655bbd63dec7695e4b5e86e0\"\n },\n {\n \"id\": \"ERR-H-L523-PARKED-DOMAINS-STAGE6-TEMP-HTTP-ROOT-VALIDATION-BUG-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 523,\n \"title\": \"PARKED_DOMAINS_STAGE6_TEMP_HTTP_ROOT_VALIDATION_BUG_20260701\",\n \"summary\": \"- Context: parked-domain HTTP-01 fixed apply proof 636. - Issue: temporary HTTP-01 route validation checked HTTP `/` for parked marker, but expected behavior before certificates is root HTTP 301 to HTTPS. - Observed: LOCAL_HTTP_VALIDATE returned HTTP_CODE=301 MARKER_OK=no, script rolled back its temporary route configs and did not run certbot. - Impact: local parked page remained OK; public HTTPS remained not closed. - Rule: validate `/.well-known/acme-challenge/` for HTTP-01 readiness, not HTTP `/`; also scan all active NPMplus nginx config dirs for unmanaged domain conflicts before writing routes.\",\n \"rule_like\": false,\n \"signature\": \"dfb310ca0c4e8c9d03ea17ec7e7c5821f16f7dde5c3b0d217d1ed361f61a8740\"\n },\n {\n \"id\": \"ERR-H-L530-PARKED-DOMAINS-STAGE7-NEWFI-ROOT-ACME-REDIRECT-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 530,\n \"title\": \"PARKED_DOMAINS_STAGE7_NEWFI_ROOT_ACME_REDIRECT_20260701\",\n \"summary\": \"- Context: parked domains HTTP-01 Stage7 stopped before certbot because ACME validation path returned 301 for root newfi.ru while www.newfi.ru, hapusya.ru, www.hapusya.ru, kingofwolk.ru and www.kingofwolk.ru returned 200. - Impact: no final parked-domain HTTPS routes/certificates were applied by Stage7; temporary route configs were rolled back. - Rule: do not run another apply stage until active NPMplus nginx route selection for newfi.ru is diagnosed with nginx -T and Host-header probes.\",\n \"rule_like\": true,\n \"signature\": \"65ab366077a7e1afb168ff94f8b1b7a8b788e1bdca76e6484300ce76c03431f9\"\n },\n {\n \"id\": \"ERR-H-L535-PARKED-DOMAINS-STAGE8-DEFAULT-HTTP-TO-HTTPS-REDIRECT-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 535,\n \"title\": \"PARKED_DOMAINS_STAGE8_DEFAULT_HTTP_TO_HTTPS_REDIRECT_20260701\",\n \"summary\": \"- Context: parked-domain route autopsy proof 638. - Finding: no active NPMplus config contains newfi.ru/hapusya.ru/kingofwolk.ru, but unknown HTTP hosts and ACME paths return 301 to HTTPS from NPMplus default routing. - Impact: HTTP-01 cannot work through the default server. A dedicated temporary ACME server block must be proven before certbot; do not run another certbot apply without a successful ACME-path probe for all six hostnames. - Rule: after every parked-domain temp route attempt, rollback the route and prove rollback before continuing.\",\n \"rule_like\": true,\n \"signature\": \"fb5de90cf615657074d466c74a748ae250def09fa8ad84bedafdc765a6682be2\"\n },\n {\n \"id\": \"ERR-H-L541-PARKED-DOMAINS-STAGE9-NEWFI-EXACT-PROBE-NEEDED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 541,\n \"title\": \"PARKED_DOMAINS_STAGE9_NEWFI_EXACT_PROBE_NEEDED_20260701\",\n \"summary\": \"- Context: parked domains HTTP-01 probe stage9 installed one combined temporary server block for all six hostnames. - Issue: five hostnames returned ACME probe 200, but root newfi.ru still returned 301. - Impact: do not continue certbot/apply until exact newfi.ru route behavior is isolated. - Rule: run a temporary exact newfi.ru probe block and roll it back before any HTTP-01 apply.\",\n \"rule_like\": true,\n \"signature\": \"b6335dafc9b854cc8c775bb66b93a4f927e15e4d1b50bb1b33eed285902c43ac\"\n },\n {\n \"id\": \"ERR-H-L547-PARKED-DOMAINS-STAGE10-EXACT-NEWFI-PROBE-OK-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 547,\n \"title\": \"PARKED_DOMAINS_STAGE10_EXACT_NEWFI_PROBE_OK_20260701\",\n \"summary\": \"- Context: parked-domain Stage10 proof 640. - Result: exact server block for newfi.ru on NPMplus intercepted both root and ACME paths with HTTP 200 and custom X-Parked-Probe markers. - Stage9 combined server_name block passed 5/6 but failed bare newfi.ru, so final HTTP-01 cutover must use separate exact per-host server blocks. - Rule: do not use one combined parked-domain server_name block for these domains; use exact host blocks and content-specific validation.\",\n \"rule_like\": true,\n \"signature\": \"6c28ff0028785a70150cd3be2574ea8ed678ed0e5d8838b93feef353f8d26e7b\"\n },\n {\n \"id\": \"ERR-H-L553-PARKED-DOMAINS-STAGE11-LOW-ID-EXACT-ROUTE-FAILED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 553,\n \"title\": \"PARKED_DOMAINS_STAGE11_LOW_ID_EXACT_ROUTE_FAILED_20260701\",\n \"summary\": \"- Context: parked-domain Stage11 proof 641. - Result: exact per-host route files 700-705 passed nginx -t but bare newfi.ru still hit HTTP->HTTPS redirect on ACME path. - Earlier Stage10 proof showed exact newfi.ru route works when placed at high id 998. - Hypothesis: file/include priority or NPMplus generated ordering makes low id 700 unsuitable for bare newfi.ru in this runtime. - Rule: do not retry parked-domain HTTP-01 with low id 700-705. First prove fixed high-id exact blocks, then apply with those exact ids.\",\n \"rule_like\": true,\n \"signature\": \"11a87298deaf78b478c1a13abcfbfb1cc91f2540eb4fc5fd7daaf3a53b426cf8\"\n },\n {\n \"id\": \"ERR-H-L560-PARKED-DOMAINS-SWITCH-TO-DNS01-AFTER-HTTP01-FAILURES-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 560,\n \"title\": \"PARKED_DOMAINS_SWITCH_TO_DNS01_AFTER_HTTP01_FAILURES_20260701\",\n \"summary\": \"- Context: parked domains newfi.ru, hapusya.ru and kingofwolk.ru. - HTTP-01 attempts hit NPMplus default HTTP-to-HTTPS redirect/include-order problems. - Stage12 cleanup confirmed temp files 980-985 absent and nginx reload OK. - Rule: do not continue HTTP-01 for these parked domains; use DNS-01 with a dedicated Cloudflare token that has Zone Read and DNS Edit for the three zones.\",\n \"rule_like\": true,\n \"signature\": \"f4fbcb9b2153251384025a82debe093241f087f381f31e999e5de5c66b60d748\"\n },\n {\n \"id\": \"ERR-H-L566-PARKED-DOMAINS-STAGE15-FALSE-LOCAL-VALIDATE-ROLLBACK-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 566,\n \"title\": \"PARKED_DOMAINS_STAGE15_FALSE_LOCAL_VALIDATE_ROLLBACK_20260701\",\n \"summary\": \"- Context: Stage15 DNS-01 succeeded and public validation showed parked page for most hosts, but local edge curl --resolve validation returned CURL_RC=60 and triggered rollback. - Impact: do not rerun Stage15 as-is. - Rule: for parked domains finalization, use existing DNS-01 certificates and validate from pve01 public HTTPS with content marker; do not use the failed local --resolve TLS check as the closure gate.\",\n \"rule_like\": true,\n \"signature\": \"3b69e4a267b02bdaf0f69e8af7da2cf979ae13223acd5a1eed82f9de1736399d\"\n },\n {\n \"id\": \"ERR-H-L571-DOMAIN-ACTIONS-STAGE18-PLACEHOLDER-AND-RENEWAL-PROOF-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 571,\n \"title\": \"DOMAIN_ACTIONS_STAGE18_PLACEHOLDER_AND_RENEWAL_PROOF_20260701\",\n \"summary\": \"- Context: after parked domains and portfolio audit, operator requested removing domain enumeration from placeholder and checking certificate renewal paths. - Rule: Stage18 must not change DNS records or NPMplus route files. It may update placeholder HTML, run existing forum renewal script once for runtime proof, and run certbot dry-run for parked certificates using existing DNS-01 hooks.\",\n \"rule_like\": true,\n \"signature\": \"ff7fb700a503f6caecb02ad5f31ed4eefe5678fd9217ef289fc3c906cd291e70\"\n },\n {\n \"id\": \"ERR-H-L575-GRAM1-ROOT-WWW-PLACEHOLDER-CLOSE-STAGE19-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 575,\n \"title\": \"GRAM1_ROOT_WWW_PLACEHOLDER_CLOSE_STAGE19_20260701\",\n \"summary\": \"- Context: operator requested gram1.ru root/www to use the existing placeholder page. - Rule: only root gram1.ru and www.gram1.ru may be routed. Existing gram1.ru subdomain NPMplus routes must not be changed. - Rule: use DNS-01 with a dedicated gram1 Cloudflare token; do not use HTTP-01. - Stage18 parked dry-run note: kingofwolk.ru dry-run hit Let's Encrypt rateLimited/service-busy after active certificate issuance and public HTTPS were already OK. Treat as transient external dry-run failure, not route/cert closure blocker.\",\n \"rule_like\": true,\n \"signature\": \"9e2adc97128fdb5159168a0ebad49e1808253dcf358cdc908303a1a29d9277b8\"\n },\n {\n \"id\": \"ERR-H-L581-PVEPRO-EDGE-LANDING-STAGE21-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 581,\n \"title\": \"PVEPRO_EDGE_LANDING_STAGE21_20260701\",\n \"summary\": \"- Context: pvepro.ru root/www currently point to the Mailcow VPS and return Mailcow with TLS hostname mismatch. - Decision: because pve01 has no SSH key access to the external VPS, do not modify the VPS. Instead move only pvepro.ru and www.pvepro.ru A records to edge and serve an edge landing page. - Safety rule: do not change mail.pvepro.ru, nb.pvepro.ru, MX, SPF, DKIM, DMARC or forum SMTP configuration. - Use a dedicated Cloudflare token for pvepro.ru. Do not print token values.\",\n \"rule_like\": true,\n \"signature\": \"da430a8849bebcbd0a5dc55d4282ae916ffb348af757e797b50a144a35c00e29\"\n },\n {\n \"id\": \"ERR-H-L587-PVEPRO-STAGE21-LANDING-SERVICE-FAILED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 587,\n \"title\": \"PVEPRO_STAGE21_LANDING_SERVICE_FAILED_20260701\",\n \"summary\": \"- Context: Stage21 stopped safely before DNS/cert/NPMplus route because local pvepro landing service returned CURL_RC=7 on 127.0.0.1:18089. - Impact: DNS root/www pvepro.ru remained on 185.139.214.215 and no route/cert changes were applied. - Rule: Stage22 must fix and prove the local landing service first, then proceed to DNS-01 cert, NPMplus route and root/www DNS only after local marker PVEPRO_LANDING_OK is reachable.\",\n \"rule_like\": false,\n \"signature\": \"6c8c70fd8a9af6ca28d7c2848c52dae8dd0405dfb2758b3341fb7d3c0f24131f\"\n },\n {\n \"id\": \"ERR-H-L592-PVEPRO-STAGE22-VALIDATION-AND-COPY-GUARD-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 592,\n \"title\": \"PVEPRO_STAGE22_VALIDATION_AND_COPY_GUARD_20260701\",\n \"summary\": \"- Context: Stage22 succeeded on edge-side landing, cert, NPMplus route and Cloudflare A upserts, but wrapper had a non-critical scp wildcard failure due __pycache__ and public validation used cached DNS for root pvepro.ru. - Mistakes recorded: do not scp scripts/* after local py_compile because __pycache__ may be present; do not continue after critical scp failure in future apply stages; do not classify PVEPro landing as NetBird just because it contains a link text; use Cloudflare authoritative A records and --resolve validation when DNS propagation may lag. - Stage23 must be read-only for infrastructure and may close only if edge route, Cloudflare A records, HTTPS landing marker, and mail/nb unchanged checks pass.\",\n \"rule_like\": true,\n \"signature\": \"39b7838d47fba86c663aac0896edc9db58fa614ebdcdbae09fb3054a061ba44f\"\n },\n {\n \"id\": \"ERR-H-L597-TAFTAUTO-CERT-AUTORENEW-BLOCKED-NO-PRIVATE-ACCESS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 597,\n \"title\": \"TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701\",\n \"summary\": \"- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router. - Current limitation: there is no safe private remote access path to the dacha router yet. - Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet. - Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists.\",\n \"rule_like\": true,\n \"signature\": \"9cd06e064f5a86be9626c4f9a2a819b8d0e71b03309c705fe0e34e1e59f81b73\"\n },\n {\n \"id\": \"ERR-H-L603-TAFTAUTO-CERT-AUTORENEW-BLOCKED-NO-PRIVATE-ACCESS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 603,\n \"title\": \"TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701\",\n \"summary\": \"- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router. - Current limitation: there is no safe private remote access path to the dacha router yet. - Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet. - Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists.\",\n \"rule_like\": true,\n \"signature\": \"9cd06e064f5a86be9626c4f9a2a819b8d0e71b03309c705fe0e34e1e59f81b73\"\n },\n {\n \"id\": \"ERR-H-L609-TAFTAUTO-WG-STAGE24-SECRET-AND-IMPORT-RULE-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 609,\n \"title\": \"TAFTAUTO_WG_STAGE24_SECRET_AND_IMPORT_RULE_20260701\",\n \"summary\": \"- Context: configuring private management path for taftauto.ru dacha router. - Rule: do not print WireGuard private keys, PSK, or client config body in proof/chat. - Rule: generated client config may be stored only as root/debian mode 600 file and must be imported into the Netcraze router UI manually. - Rule: do not open router public admin or certificate deployment until WireGuard handshake is proven.\",\n \"rule_like\": true,\n \"signature\": \"6722ad8ce177f600b198fb48046657acb926330bfbd8bbe1c810de69015f5a87\"\n },\n {\n \"id\": \"ERR-H-L615-TAFTAUTO-WG-PSK-ROTATION-SCRIPT-BROKE-DATAPLANE-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 615,\n \"title\": \"TAFTAUTO_WG_PSK_ROTATION_SCRIPT_BROKE_DATAPLANE_20260702\",\n \"summary\": \"- Assistant gave unsafe PSK rotation flow; after profile reimport dacha interface changed from Wireguard0 to Wireguard1 and data-plane broke until security-level, ACL and routes were restored on Wireguard1. - Proof 666 is superseded because it showed PUBLIC_SSH_22_STILL_OPEN. - Final fixed proof: /root/evidence/667_TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702_PROOF.txt\",\n \"rule_like\": false,\n \"signature\": \"4b046c53fa6020f8c97bd7e00a8a7d78280d3ab2bf337816cb559083ba9c7495\"\n },\n {\n \"id\": \"ERR-H-L620-TAFTAUTO-CERTBOT-CLOUDFLARE-PLUGIN-MISSING-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 620,\n \"title\": \"TAFTAUTO_CERTBOT_CLOUDFLARE_PLUGIN_MISSING_20260702\",\n \"summary\": \"- Certbot on edge rejected --dns-cloudflare-* arguments: dns-cloudflare plugin missing/not loaded. - No taftauto certificate issued in failed run; no DNS A records or NPMplus routes changed.\",\n \"rule_like\": false,\n \"signature\": \"3e7ad3e9ef090a52212334ce2d8c5b65fa7dae68481c01068ec7845bef14a07f\"\n },\n {\n \"id\": \"ERR-H-L624-HOMEPAGE-SERVICES-YAML-BAD-INDENT-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 624,\n \"title\": \"HOMEPAGE_SERVICES_YAML_BAD_INDENT_20260702\",\n \"summary\": \"- Assistant inserted Homepage services.yaml entries with wrong indentation; YAMLException bad indentation at line 227. - Fix: rebuild SmartApe under External Homelabs with 2-space item indentation, remove Router Moscow, rebuild Sites category.\",\n \"rule_like\": false,\n \"signature\": \"da1ecebf560240f444c3df39efb068b9d7c8ab7ceda2053516e8d82c8567bd7b\"\n },\n {\n \"id\": \"ERR-H-L628-HOMEPAGE-SERVICES-YAML-BAD-INDENT-REPAIR-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 628,\n \"title\": \"HOMEPAGE_SERVICES_YAML_BAD_INDENT_REPAIR_20260702\",\n \"summary\": \"- Previous apply broke services.yaml indentation and did not follow YAML-aware rule. - Repair uses latest valid services.yaml backup, PyYAML parse/dump, validates before restart.\",\n \"rule_like\": false,\n \"signature\": \"7b789067b17326a4ead14942608a43b86b4ae971896eff93eb8658684e657e22\"\n },\n {\n \"id\": \"ERR-H-L632-TAFTAUTO-CERTBOT-DRYRUN-BAD-FLAG-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 632,\n \"title\": \"TAFTAUTO_CERTBOT_DRYRUN_BAD_FLAG_20260702\",\n \"summary\": \"- Assistant used unsupported certbot flag --manual-public-ip-logging-ok with certbot 4.0.0 during taftauto renew dry-run. - No certificate, DNS, NPMplus route, or deploy state changed by the failed dry-run.\",\n \"rule_like\": false,\n \"signature\": \"518dbc241694b2eb7c4631a1b147a8262af537e6f4eaf082d1f092cc0a8087cb\"\n },\n {\n \"id\": \"ERR-H-L636-TAFTAUTO-CERT-RENEW-DRYRUN-RATE-LIMITED-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 636,\n \"title\": \"TAFTAUTO_CERT_RENEW_DRYRUN_RATE_LIMITED_20260702\",\n \"summary\": \"- Proof 672 is superseded/partial: it printed STATUS=OK even though certbot dry-run failed. - Failure reason: Let's Encrypt staging/service returned rateLimited / Service busy; retry later. - Cloudflare manual auth and cleanup hooks did run successfully. - Deploy hook was installed and manually invoked successfully before the dry-run. - Do not retry immediately.\",\n \"rule_like\": true,\n \"signature\": \"15452c7cc7d791a7cc8324a5f2d39544b8a0971fce736a46afdfce7ec8eea5b3\"\n },\n {\n \"id\": \"ERR-H-L643-HOMEPAGE-XENFORO-INFO-INLINE-PYTHON-SYNTAX-ERROR-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 643,\n \"title\": \"HOMEPAGE_XENFORO_INFO_INLINE_PYTHON_SYNTAX_ERROR_20260702\",\n \"summary\": \"- Assistant used invalid inline Python syntax with def after semicolon while adding XenForo.info to Homepage External. - Failed before services.yaml write.\",\n \"rule_like\": false,\n \"signature\": \"3620b01df453a95e2e7fbd121ee954690344e32ed12c71b84739e1a554b316d2\"\n },\n {\n \"id\": \"ERR-H-L647-HOMEPAGE-USEFUL-ROUTER-NAME-MISMATCH-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 647,\n \"title\": \"HOMEPAGE_USEFUL_ROUTER_NAME_MISMATCH_20260702\",\n \"summary\": \"- Previous apply looked for Russian Router card name, but active Homepage Useful group contains Router and Public Domain. - Failed before write. Fix uses YAML-aware edit with exact names Router/Public Domain.\",\n \"rule_like\": false,\n \"signature\": \"49212250bc5eea53887790a1782ea3acdc2fa2e7769d89ee79f603883533adf8\"\n },\n {\n \"id\": \"ERR-H-L651-HOMEPAGE-USEFUL-ROUTER-CARD-NOT-FOUND-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 651,\n \"title\": \"HOMEPAGE_USEFUL_ROUTER_CARD_NOT_FOUND_20260702\",\n \"summary\": \"- Apply expected Useful -> Router in active services.yaml, but YAML parse did not find it. No write was performed. - Need verify active Homepage config path, docker mount, and any alternate services.yaml before next apply.\",\n \"rule_like\": false,\n \"signature\": \"ce9a93f7876b27d64e76cddc59b220166fa0e82ab7265d3373352906d6e10264\"\n },\n {\n \"id\": \"ERR-H-L655-20260702-CF-TOKEN-AUDIT-BROKEN-COMMAND\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 655,\n \"title\": \"20260702_CF_TOKEN_AUDIT_BROKEN_COMMAND\",\n \"summary\": \"- Ошибка: ассистент дал хрупкую команду аудита Cloudflare-токенов с вложенным heredoc/Python через ssh. - Факт: bash начал интерпретировать Python-код, появились syntax error и вторичный prompt >. - Правило: не давать nested heredoc/сложные кавычки через ssh; для длинной логики сначала класть скрипт отдельным файлом, затем запускать его. - Статус: аудит токенов не выполнен, инфраструктурные изменения не должны считаться применёнными.\",\n \"rule_like\": false,\n \"signature\": \"00c3ff174b7bd8af54a071aad893fc3e28aae0f79b8cba6babffa67869d0d7c3\"\n },\n {\n \"id\": \"ERR-H-L661-LESSON-20260702-CROWDSEC-NETBIRD-EXIT-ROUTE\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 661,\n \"title\": \"LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE\",\n \"summary\": \"- Do not treat e3qxxx.netbird.selfhosted / 100.100.125.70 and e3qxxx-183-106.netbird.selfhosted / 100.100.183.106 as VPS egress nodes; they are user mobile peers. - Correct egress candidates are relay.pvepro.ru / relay.netbird.selfhosted / 100.100.19.1 for Moldova and mail.pvepro.ru / mail.netbird.selfhosted / 100.100.147.204 for USA. - Access Policy edge-vm -> relay/mail only proves peer reachability. It is not an internet exit route. - Required NetBird fix for CrowdSec CAPI was Network Routing exit route 0.0.0.0/0 distributed to edge-vm group, routing peer relay, masquerade/NAT enabled. - Before attempting CrowdSec CAPI, verify on edge-vm: ip route get 1.1.1.1 uses wt0, wg allowed-ips contains 0.0.0.0/0, and Cloudflare trace no longer shows home IP 95.84.154.183.\",\n \"rule_like\": true,\n \"signature\": \"651aa75a20b1bad649519711690f3a5d4f944e71649c31414c966c56c2647f83\"\n },\n {\n \"id\": \"ERR-H-L668-LESSON-20260702-HOMEPAGE-MONITORS-REPAIR-NOT-DELETE\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 668,\n \"title\": \"LESSON_20260702_HOMEPAGE_MONITORS_REPAIR_NOT_DELETE\",\n \"summary\": \"- Do not delete or disable Homepage siteMonitor fields to hide red badges. - Fix the underlying monitor path, DNS, proxy route, or local health-wrapper instead. - Do not touch Cloudflare Homepage card when operator says it is green and opens correctly. - Before changing Homepage, identify the live config host; in this state live Homepage is on core-apps [PRIVATE_IP], not edge-vm.\",\n \"rule_like\": true,\n \"signature\": \"2bc16e142d368e59b2b176de6779445c152df7e8bae2b1be79a2f4d075985c7c\"\n },\n {\n \"id\": \"ERR-H-L674-LESSON-20260702-KUMA-ADD-ONE-MONITOR-ONLY\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 674,\n \"title\": \"LESSON_20260702_KUMA_ADD_ONE_MONITOR_ONLY\",\n \"summary\": \"- When asked to fix one missing Uptime Kuma service, add only one monitor, then generate proposals separately. - Before direct Kuma DB mutation, stop the container and create a DB backup. - Verify DB integrity before starting Kuma again. - Do not touch Cloudflare when operator says it is green and opens correctly.\",\n \"rule_like\": true,\n \"signature\": \"b268d57842a9dd192fd72dd940879f3e7cf714ae2928a68a23b5e1ed347262f0\"\n },\n {\n \"id\": \"ERR-H-L680-LESSON-20260702-DOCKGE-STALE-STACKS-NOT-CONTAINERS\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 680,\n \"title\": \"LESSON_20260702_DOCKGE_STALE_STACKS_NOT_CONTAINERS\",\n \"summary\": \"- Dockge inactive items after migration can be stale compose folders, not stopped containers. - First classify runtime projects across all Docker hosts before deleting or archiving anything. - For remote Dockge visibility, projects must be under /opt/stacks or bind-mounted there. - Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions.\",\n \"rule_like\": true,\n \"signature\": \"4249f24d95f2186959586b18fa083cdac97ceddb1cc65b0bd0fd1eeddfdea50d\"\n },\n {\n \"id\": \"ERR-H-L686-STAGE4B-MASKED-SQL-QUERY-FAILURE-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 686,\n \"title\": \"STAGE4B_MASKED_SQL_QUERY_FAILURE_20260714\",\n \"summary\": \"- Previous Stage4B read-only audit returned empty observation-summary, metadata-key, trigger and function blocks because nested SSH/SQL quoting broke queries while stderr was hidden. - Empty blocks are query failures, not proof that metadata, triggers or functions are absent. - Future PostgreSQL audits must use ON_ERROR_STOP, visible stderr, independent query RC and quote-safe dollar literals.\",\n \"rule_like\": false,\n \"signature\": \"9786689ba74fed176972e94389a37a7664ee81618201ff24852199ed754b694f\"\n },\n {\n \"id\": \"ERR-H-L691-STAGE4C-SCHEMA-MIGRATIONS-ID-ASSUMPTION-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 691,\n \"title\": \"STAGE4C_SCHEMA_MIGRATIONS_ID_ASSUMPTION_20260714\",\n \"summary\": \"- Stage4C read-only preflight incorrectly assumed schema_migrations has a column named id. - PostgreSQL returned column id does not exist and ON_ERROR_STOP correctly stopped all following SQL checks. - Production DB and application were not changed. Future migration audits must inspect information_schema first and query migration rows without assuming column names.\",\n \"rule_like\": false,\n \"signature\": \"f166dab2251442df1f0e77fb70dfa166f1e681a94567cc0d17eb7e5000466a11\"\n },\n {\n \"id\": \"ERR-H-L696-STAGE4C-PGDUMP-DEV-NULL-FSYNC-INVALID-PROBE-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 696,\n \"title\": \"STAGE4C_PGDUMP_DEV_NULL_FSYNC_INVALID_PROBE_20260714\",\n \"summary\": \"- Stage4C custom-format pg_dump capability probe incorrectly used /dev/null as the output file. - pg_dump failed only because fsync on /dev/null returned Invalid argument. - Production database and application were not changed. - Future custom dump capability checks must use a real temporary regular file, validate it with pg_restore --list, and remove it afterward.\",\n \"rule_like\": false,\n \"signature\": \"c565b0d0dc04089e68783eb6f6b52e7e66fd66b73eb84b37ea6d7f324ca2142c\"\n },\n {\n \"id\": \"ERR-H-L702-STAGE4C-REMOTE-PREP-THIRD-SUBSTEP-FAILED-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 702,\n \"title\": \"STAGE4C_REMOTE_PREP_THIRD_SUBSTEP_FAILED_20260714\",\n \"summary\": \"- Isolated migration dry-run did not start: REMOTE_PREP_RC=0|0|1. - Remote directory creation and SCP succeeded; the third ownership/mode preparation substep returned RC=1. - Production database and application were not changed. - Exact cause requires read-only residual directory and permission inspection before cleanup or retry.\",\n \"rule_like\": false,\n \"signature\": \"c048e5da3ee1467f7b4742ae2ef671944078c985979c4c75f87aefe2b13c54ad\"\n },\n {\n \"id\": \"ERR-H-L708-STAGE4C-REMOTE-PREP-GLOB-AFTER-CHMOD-ROOT-CAUSE-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 708,\n \"title\": \"STAGE4C_REMOTE_PREP_GLOB_AFTER_CHMOD_ROOT_CAUSE_20260714\",\n \"summary\": \"- Exact cause of REMOTE_PREP_RC=0|0|1: after the directory became postgres-owned mode 0700, the unprivileged debian shell could not traverse it and could not expand the wildcard used by chmod. - Candidate files and SHA256 verification were valid; no temporary database was created and production was unchanged. - Future preparation must use exact privileged paths or a verified shared group, never an unprivileged wildcard after restrictive chmod.\",\n \"rule_like\": true,\n \"signature\": \"a174131243eb8db72dedb89bc3e917cb45e7c7c8107e40c90b476d46bf4d2718\"\n },\n {\n \"id\": \"ERR-H-L713-STAGE4C-MIGRATION003-FINGERPRINT-DOLLAR-QUOTE-FAILURE-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 713,\n \"title\": \"STAGE4C_MIGRATION003_FINGERPRINT_DOLLAR_QUOTE_FAILURE_20260714\",\n \"summary\": \"- Migration 003 isolated dry-run failed at fingerprint CHECK because the regex end anchor collided with the SQL dollar-quote delimiter and produced an unterminated dollar-quoted string. - The migration transaction rolled back, the temporary database was removed, and production remained unchanged. - Fingerprint validation must use length plus translate without a regex dollar anchor; corrected candidate must pass a fresh isolated dry-run before production consideration.\",\n \"rule_like\": false,\n \"signature\": \"adc9a3f845fb0a58a7d9e0196174f3a9c0b5eb905bbd2c2eb456a92eb9f2d37b\"\n },\n {\n \"id\": \"ERR-H-L718-STAGE4C-SEAL-OUTER-RC-MASKING-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 718,\n \"title\": \"STAGE4C_SEAL_OUTER_RC_MASKING_20260714\",\n \"summary\": \"- Stage4C final seal correctly refused creation because overall health was WARN, returning internal RC=40. - The former wrapper then printed a second misleading COMMAND_RC=0 because top-level true masked the internal result. - Future commands must capture the guarded subshell through PIPESTATUS and print exactly one authoritative COMMAND_RC before the mandatory final true. - Production database, application and services were unchanged.\",\n \"rule_like\": false,\n \"signature\": \"9f71003c157467e9b2bc0405d3137b6f0f7d14e053cb3053e9323219494287a9\"\n },\n {\n \"id\": \"ERR-H-L724-STAGE4C-DIAGNOSIS-EXPECTED-DERIVED-FILE-ON-VM180-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 724,\n \"title\": \"STAGE4C_DIAGNOSIS_EXPECTED_DERIVED_FILE_ON_VM180_20260714\",\n \"summary\": \"- Previous diagnosis incorrectly treated cluster-admin-restricted-probes.txt as a required source file on VM180. - The file is a central derived health artifact on pve01; VM180 supplies full-observer evidence instead. - This false expectation caused diagnostic RC=32 without any infrastructure failure or production change.\",\n \"rule_like\": false,\n \"signature\": \"d032d83a642f525e4ef82024ba9f810d8b9cbb88a6351da3dbed17391e14056d\"\n },\n {\n \"id\": \"ERR-H-L729-STAGE4C-NO-JOBS-RUNNING-TEXT-COUNTED-AS-JOB-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 729,\n \"title\": \"STAGE4C_NO_JOBS_RUNNING_TEXT_COUNTED_AS_JOB_20260714\",\n \"summary\": \"- Stage4C seal preflight incorrectly counted the systemctl informational text No jobs running as one active job. - The parser counted every nonempty output line instead of accepting only rows beginning with a numeric systemd Job ID. - No service was started and no production state changed during the blocked attempt. - Future job counts must match a numeric first field only.\",\n \"rule_like\": false,\n \"signature\": \"619b903f2e18ba4251959e200fe4511995e56c4cd9a3e63b5fc8d77134df5ade\"\n },\n {\n \"id\": \"ERR-H-L735-STAGE4D-ADAPTER-SOURCE-AUDIT-ABORTED-BEFORE-FINAL-MARKERS-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 735,\n \"title\": \"STAGE4D_ADAPTER_SOURCE_AUDIT_ABORTED_BEFORE_FINAL_MARKERS_20260714\",\n \"summary\": \"- The evidence-only Stage4D adapter source audit returned RC=1 before emitting its VM180 completion and final audit markers. - The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults. - Production, database, application, services, timers, health and desired-state were unchanged. - Inspect the preserved runner stdout, stderr and result JSON before correcting or rerunning the task.\",\n \"rule_like\": true,\n \"signature\": \"cc2c72b5d6eb7a0a1f3258050f4509d2e08f538236449d0100a9ee45193da58d\"\n },\n {\n \"id\": \"ERR-H-L741-STAGE4D-LOCAL-EXPANSION-OF-REMOTE-Q-UNDER-NOUNSET-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 741,\n \"title\": \"STAGE4D_LOCAL_EXPANSION_OF_REMOTE_Q_UNDER_NOUNSET_20260714\",\n \"summary\": \"- The first Stage4D adapter source audit embedded a large remote program inside a locally double-quoted SSH argument. - Local shell expansion reached the remote variable Q while nounset was enabled, causing Q: unbound variable before SSH execution. - VM180 and PostgreSQL audits did not start; production state was unchanged. - Remote shell and SQL programs must be separate syntax-checked files passed through SSH stdin.\",\n \"rule_like\": false,\n \"signature\": \"bdae2b3ceda91c697838a15c35c327d73583cc201d5fc77c2f7355f45c2e340a\"\n },\n {\n \"id\": \"ERR-H-L747-STAGE4D-REMOTE-PYTHON311-ADAPTER-SELFTEST-RC1-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 747,\n \"title\": \"STAGE4D_REMOTE_PYTHON311_ADAPTER_SELFTEST_RC1_20260714\",\n \"summary\": \"- Canonical adapter candidate passed local Python 3.13 self-tests with 48 envelopes but returned RC=1 on VM180 Python 3.11.2. - Remote upload and cleanup succeeded, and production database remained 0|0|OK. - The exact traceback and failing assertion must be captured by an isolated identical reproduction before changing the candidate.\",\n \"rule_like\": false,\n \"signature\": \"6111533bbd96a60d54d47dff09af497c706e31b90977efb40efc8d5c24bf5ada\"\n },\n {\n \"id\": \"ERR-H-L752-STAGE4D-PYTHON311-FSTRING-COMPATIBILITY-AND-REPRO-VALIDATOR-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 752,\n \"title\": \"STAGE4D_PYTHON311_FSTRING_COMPATIBILITY_AND_REPRO_VALIDATOR_20260714\",\n \"summary\": \"- Adapter candidate passed Python 3.13 but failed Python 3.11 parsing at Path(row[\\\"path\\\"]) inside a double-quoted f-string. - Exact fix is Python 3.11-compatible quoting: Path(row['path']). - The reproduction command captured the expected compile failure but incorrectly returned CAPTURE_BAD=1 because its validator required compile RC=0. - Production database remained 0|0|OK and desired-state remained clean. - Future adapter candidates must pass syntax and full self-tests on both pve01 and VM180 runtimes before acceptance.\",\n \"rule_like\": false,\n \"signature\": \"5aec931e3d68bbdfd0f84e397c8ad1aea4fe7e9d239f066d7186823aaf5a0c33\"\n },\n {\n \"id\": \"ERR-H-L759-STAGE4E-DESIGN-VALIDATOR-ABORTED-BEFORE-LOCAL-VALIDATION-MARKER-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 759,\n \"title\": \"STAGE4E_DESIGN_VALIDATOR_ABORTED_BEFORE_LOCAL_VALIDATION_MARKER_20260714\",\n \"summary\": \"- Restricted probe-agent design candidate compiled successfully but its local validator exited before producing LOCAL_VALIDATION_RC. - The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults. - Active external probes were not executed and production state was unchanged. - Inspect the preserved validator traceback and exact assertion before modifying the candidate.\",\n \"rule_like\": false,\n \"signature\": \"fa47b366640ad2b68b1efc2d897808d7e75322d7aa9e3c9af89013e4eecc5a26\"\n },\n {\n \"id\": \"ERR-H-L765-STAGE4E-VALIDATOR-SCANNED-PYC-AND-OWN-FORBIDDEN-LITERALS-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 765,\n \"title\": \"STAGE4E_VALIDATOR_SCANNED_PYC_AND_OWN_FORBIDDEN_LITERALS_20260714\",\n \"summary\": \"- Stage4E design validator recursively read every candidate file as UTF-8 after py_compile had created binary __pycache__ bytecode. - This caused UnicodeDecodeError before any design assertion failed. - The phase-wide forbidden-pattern scan also included validate_design.py itself, which necessarily contains the prohibited literals it verifies. - Validators must use an explicit text-file allowlist and scans must exclude validator implementation and binary artifacts. - Production, database, services, timers and desired-state were unchanged; active external probes were not executed.\",\n \"rule_like\": true,\n \"signature\": \"76b2888a1b774c0f7e8c0fe3aab0e3beb73c78f9b46ef3694f4708cc3431230c\"\n },\n {\n \"id\": \"ERR-H-L772-STAGE4E-V2-VALIDATOR-FAILED-STATIC-COMPILE-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 772,\n \"title\": \"STAGE4E_V2_VALIDATOR_FAILED_STATIC_COMPILE_20260714\",\n \"summary\": \"- Stage4E design candidate v2 failed static compilation before local design validation started. - The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects. - The generated validator must be inspected at the exact SyntaxError line before another candidate is created. - Production, database, application, services, timers, health and desired-state were unchanged; active external probes were not executed.\",\n \"rule_like\": false,\n \"signature\": \"0acd318ec69af959e23233f89a6b26cefcce1dbb2c85908fcc2406c51c85db72\"\n },\n {\n \"id\": \"ERR-H-L778-STAGE4E-PIPESTATUS-LOST-AFTER-FIRST-ASSIGNMENT-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 778,\n \"title\": \"STAGE4E_PIPESTATUS_LOST_AFTER_FIRST_ASSIGNMENT_20260714\",\n \"summary\": \"- VM180 validation copy used a pipeline and read PIPESTATUS in two separate assignments. - The first assignment reset PIPESTATUS, leaving COPY_RC empty and preventing remote validation and cleanup. - Retry must use explicit SCP operations without pipeline status parsing. - Production remained unchanged and active external probes were not executed.\",\n \"rule_like\": false,\n \"signature\": \"29d5bdfa098352aeceb9defce46f9845d100c46dfc1758f17ad8c5eb56859787\"\n },\n {\n \"id\": \"ERR-H-L784-STAGE4F-VALIDATOR-EXPECTED-HEALTH-REFRESH-FROM-NONWRITING-BACKUP-SCRIPT-\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 784,\n \"title\": \"STAGE4F_VALIDATOR_EXPECTED_HEALTH_REFRESH_FROM_NONWRITING_BACKUP_SCRIPT_20260714\",\n \"summary\": \"- The controlled backup service completed with Result=success and ExecMainStatus=0. - Validation incorrectly required cluster-admin-incident-engine-backup.txt to receive a fresh timestamp. - The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure. - Stage4F backup proof must use the service result, fresh latest JSON, produced artifacts and off-host/restore evidence instead. - The backup must not be rerun merely to satisfy the invalid health-file freshness assertion.\",\n \"rule_like\": true,\n \"signature\": \"92516bc4ce3841e809e6ec04f2ec77458c1c472c146af37a577bbeffbce5d19d\"\n },\n {\n \"id\": \"ERR-H-L791-STRICT-RULE-20260714-CLOSE-TAILS-IMMEDIATELY\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 791,\n \"title\": \"STRICT_RULE_20260714_CLOSE_TAILS_IMMEDIATELY\",\n \"summary\": \"- Failure class: переход к следующей задаче при наличии незакрытого хвоста. - Mandatory anti-regression: после ошибки запрещено продолжать другой scope до точной root cause, исправления, повторной проверки, очистки, proof и seal. - Closure gate: UNRESOLVED_TAIL_COUNT=0; BLOCKER_COUNT=0; TEMPORARY_ARTIFACT_COUNT=0; ROLLBACK_VERIFIED=YES; HEALTH=OK; PROOF_UPDATED=YES; REFERENCE_UPDATED=YES. - Если закрытие невозможно из-за внешней зависимости, опасной неоднозначности или обязательного решения пользователя, фиксировать BLOCKED/OPEN и не заявлять CLOSED. - Proof: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/report.txt\",\n \"rule_like\": true,\n \"signature\": \"951ac14651bd989701f8f0cdf4a6f683fdbe957a357523c878cef0081fdc8dd7\"\n },\n {\n \"id\": \"ERR-H-L798-ERROR-20260714-DEPENDENCY-FACT-QUERY-GLOBAL-LIMIT\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 798,\n \"title\": \"ERROR_20260714_DEPENDENCY_FACT_QUERY_GLOBAL_LIMIT\",\n \"summary\": \"- Symptom: dependency audit вернул только latest_collector_status. - Root cause: ORDER BY 1 LIMIT 1 в конце UNION ALL применился ко всему набору фактов. - Correction: latest collector status извлекается скалярным подзапросом внутри CTE facts; глобальный LIMIT отсутствует. - Anti-regression: multi-fact SQL audit обязан проверять точное ожидаемое число строк до извлечения значений. - Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt - Status: CLOSED.\",\n \"rule_like\": false,\n \"signature\": \"8078498e8c9915662cb7bbe485ccb1e71099a6a8d39849830721e2bcd69fde70\"\n },\n {\n \"id\": \"ERR-H-L806-ERROR-20260714-COLLECTOR-PATCH-REQUIREMENT-PATTERN-MISMATCH\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 806,\n \"title\": \"ERROR_20260714_COLLECTOR_PATCH_REQUIREMENT_PATTERN_MISMATCH\",\n \"summary\": \"- Symptom: MIGRATION_PLAN_COLLECTOR_PATCH_REQUIRED_COUNT=1 при ожидаемых двух подтверждениях. - Root cause: узкий шаблон учитывал collector_patch_required, но не collector_patch_required_after_stage4c. - Correction: оба поля проверяются отдельно, затем складываются как semantic requirement count. - Anti-regression: связанные JSON-контракты проверяются по собственным точным именам полей, а не одним унифицированным grep-шаблоном. - Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt - Status: CLOSED; PRIOR_TAIL_UNRESOLVED_COUNT=0.\",\n \"rule_like\": false,\n \"signature\": \"a136bb3af68dd9abefd3054a299cbe72d1ee4435d755299318badd0469472379\"\n },\n {\n \"id\": \"ERR-H-L814-ERROR-20260714-PRIVILEGED-FILE-REDIRECTION-BEFORE-SUDO\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 814,\n \"title\": \"ERROR_20260714_PRIVILEGED_FILE_REDIRECTION_BEFORE_SUDO\",\n \"summary\": \"- Symptom: bash reported Permission denied while counting collector.py lines. - Root cause: `< collector.py` was opened by the unprivileged caller shell before sudo executed wc. - Correction: run sudo wc -l collector.py without caller-side input redirection. - Anti-regression: never combine sudo with `< protected-file`; pass the protected path as an argument to the privileged command. - Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt - Status: CLOSED.\",\n \"rule_like\": true,\n \"signature\": \"4510dad88d4639e1987b83002657a7c5c4580d0b4ae4b398c376b2ca49aa394d\"\n },\n {\n \"id\": \"ERR-H-L822-ERROR-20260714-COLLECTOR-RUN-COLUMN-ASSUMPTION\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 822,\n \"title\": \"ERROR_20260714_COLLECTOR_RUN_COLUMN_ASSUMPTION\",\n \"summary\": \"- Symptom: SQL failed because completed_at did not exist. - Root cause: collector_runs columns were assumed instead of read from information_schema; actual names are finished_at and error_text. - Correction: assert required and forbidden column counts before querying recent runs. - Anti-regression: every schema-sensitive audit must discover and validate exact column names before the main query. - Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": true,\n \"signature\": \"01f861530e0df5cbb6814d21fe0caa2043d2819342012f74d4299c45bfcf08cd\"\n },\n {\n \"id\": \"ERR-H-L830-ERROR-20260714-REMOTE-STDIN-ARGUMENT-SHIFT\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 830,\n \"title\": \"ERROR_20260714_REMOTE_STDIN_ARGUMENT_SHIFT\",\n \"summary\": \"- Symptom: remote harness выполнил chmod для пути bash. - Root cause: дополнительный positional token bash был передан после SSH command и стал первым аргументом stdin-скрипта. - Correction: remote command вызывает bash -s -- с ровно тремя явно встроенными аргументами: candidate path, SHA256 и byte count. - Anti-regression: stdin-скрипт обязан валидировать $1 по разрешённому path pattern и затем подтверждать точные SHA256 и byte count; запрещено добавлять отдельный bash-token после remote command. - Closure proof: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/report.txt - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": true,\n \"signature\": \"a808754bcc91814fedf38578e86631a14e4964c3ab9571e205b6b65afa951389\"\n },\n {\n \"id\": \"ERR-H-L838-ERROR-20260714-OVERSIZED-COMMAND-PARSE-FAILURE\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 838,\n \"title\": \"ERROR_20260714_OVERSIZED_COMMAND_PARSE_FAILURE\",\n \"summary\": \"- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`. - Root cause: чрезмерно длинная интерактивная команда была обрезана или повреждена при передаче до полного разбора Bash. - Impact: side-effect audit подтвердил production state 0|0|OK, неизменный collector, активный timer и отсутствие временных БД, каталогов и процессов. - Correction: монолитные команды запрещены; сложные действия выполняются через отдельный проверяемый task-скрипт и короткий launcher. - Anti-regression: interactive payload target <= 8000 bytes; syntax-check перед запуском; remote output сохраняется независимо от RC. - Closure proof: /var/lib/homelab-change-sets/operator-command-size-guard-v1/20260714T182653Z/report.txt - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": false,\n \"signature\": \"16b0958121cab8d184e16e7ea30930971bc785f4385caaeacea19bb35a8e8908\"\n },\n {\n \"id\": \"ERR-H-L847-ERROR-20260714-HOMELAB-ADMIN-HELP-RC-ASSUMPTION\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 847,\n \"title\": \"ERROR_20260714_HOMELAB_ADMIN_HELP_RC_ASSUMPTION\",\n \"summary\": \"- Symptom: template audit завершился HOMELAB_ADMIN_HELP_FAILED. - Root cause: был ошибочно ожидаем RC=0 от неподдерживаемого аргумента --help. - Actual contract: неизвестный аргумент печатает usage и возвращает RC=64. - Correction: проверять version либо считать точную usage-строку вместе с RC=64 штатным результатом. - Anti-regression: перед проверкой CLI сначала фиксировать поддерживаемые команды; не предполагать GNU-style --help. - Closure proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v1/20260714T183338Z/report.txt - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": false,\n \"signature\": \"346a85cc8494d14ed401b27e57d0b73a8d53078fdd91575ccb4dc89e833d4e44\"\n },\n {\n \"id\": \"ERR-H-L856-ERROR-20260715-EXPECTED-NEGATIVE-RC-TRAP-CLASSIFICATION\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 856,\n \"title\": \"ERROR_20260715_EXPECTED_NEGATIVE_RC_TRAP_CLASSIFICATION\",\n \"summary\": \"- Symptom: schema-v2 task v3 build stopped while intentionally testing the --invalid argument path. - Root cause: an expected RC64 was executed while the generic ERR trap remained active. - Correction: expected failures are evaluated only through an explicit if/else branch with immediate RC capture. ANTI_REGRESSION_TOKEN=EXPECTED_RC64_INTERCEPTED_ERR_TRAP - Required pattern: if command; then rc=0; else rc=$?; fi, followed by a separate exact expected-RC assertion. - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": false,\n \"signature\": \"99b412bad31ee6959b473695ef09baa07b43e09ded03a4a254d60c25715f6059\"\n },\n {\n \"id\": \"ERR-H-L864-ERROR-20260715-DUPLICATE-MACHINE-MARKER-COUNT\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 864,\n \"title\": \"ERROR_20260715_DUPLICATE_MACHINE_MARKER_COUNT\",\n \"summary\": \"- Symptom: error-register candidate construction stopped with RC1 before applying the candidate. - Root cause: the same marker text appeared in both a heading and a descriptive line, while the guard expected one substring occurrence. - Correction: headings and prose do not contain machine-token values; validation counts only complete exact token lines. ANTI_REGRESSION_TOKEN=DUPLICATE_MACHINE_TOKEN_IN_HEADING_AND_BODY - Required pattern: grep -Fxc against a complete machine line, never grep -Foc against an unrestricted substring. - Production impact: none. - Temporary artifacts: removed and verified. - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": true,\n \"signature\": \"356ff9228e7bb1e236f72584597d788a4dc65367b0a1e4af38209a9b24be13ac\"\n },\n {\n \"id\": \"ERR-H-L875-ERROR-20260715-OUTER-WRAPPER-SHELL-SYNTAX\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 875,\n \"title\": \"ERROR_20260715_OUTER_WRAPPER_SHELL_SYNTAX\",\n \"summary\": \"- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis. - Root cause: the outer wrapper contained an unbalanced compound shell construct before the closing subshell delimiter. - Impact: parsing failed before the start marker, guards and homelab-admin invocation; no task phase or infrastructure action executed. - Correction: use a linear wrapper with explicitly closed if/case blocks and avoid nested brace groups in conditional expressions. ANTI_REGRESSION_TOKEN=OUTER_WRAPPER_UNBALANCED_SUBSHELL_SYNTAX - Anti-regression: after any parser-level failure, treat the attempted phase as not executed and verify package and run-state before retrying. - Production impact: none. - Task package impact: none.\",\n \"rule_like\": true,\n \"signature\": \"208c08a253c47c1ad448f2282eacc0af231ebeb3443f3f888206b3d7926d9eef\"\n },\n {\n \"id\": \"ERR-H-L888-ERROR-20260715-INLINE-REMOTE-SHELL-IN-APPLY-PHASE\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 888,\n \"title\": \"ERROR_20260715_INLINE_REMOTE_SHELL_IN_APPLY_PHASE\",\n \"summary\": \"- Symptom: homelab-admin lint rejected Stage4H task v6 with ERR-002-INLINE-REMOTE-SHELL. - Root cause: apply.sh invoked bash -s through an inline quoted SSH program and streamed the remote script through stdin. - Correction: copy the declared remote script to an isolated fixed path on VM180 and invoke that file directly with positional arguments. ANTI_REGRESSION_TOKEN=INLINE_REMOTE_SHELL_REPLACED_BY_STAGED_REMOTE_SCRIPT_PATH - Anti-regression: phase scripts may perform transport orchestration, but remote shell program bodies must exist as separately declared, syntax-checked remote script files. - Remote stdout and stderr must be preserved before evaluating the remote return code. - Task v6 status: REJECTED_BY_LINT; never executed and never mutated. - Production database impact: none.\",\n \"rule_like\": true,\n \"signature\": \"5c6fc9cd51fa9d557d4c8aee96a26b3d08bb3e86c6bd4813a7336a87c433b8bc\"\n },\n {\n \"id\": \"ERR-H-L902-ERROR-20260715-OPTIONAL-RUN-DIRECTORY-FIND-UNDER-PIPEFAIL\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 902,\n \"title\": \"ERROR_20260715_OPTIONAL_RUN_DIRECTORY_FIND_UNDER_PIPEFAIL\",\n \"summary\": \"- Symptom: task-v7 builder stopped while counting formal runs for rejected task v6. - Root cause: find was executed against an optional directory that did not exist; under pipefail the assignment returned RC1. - Correction: test directory existence first and assign zero without invoking find when it is absent. ANTI_REGRESSION_TOKEN=MISSING_OPTIONAL_RUN_DIRECTORY_TREATED_AS_ZERO - Anti-regression: optional paths must have an explicit existence branch before find under pipefail. - Production impact: none. - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. - Registered at: 2026-07-15T04:44:06Z\",\n \"rule_like\": false,\n \"signature\": \"f5b1a0f37dcba3d28adda172281f4f4b53abc87f36ec3ed7fe35cf51845f0794\"\n },\n {\n \"id\": \"ERR-H-L912-ERROR-20260715-PG-CONSTRAINT-CONTYPE-CHAR-CONCAT\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 912,\n \"title\": \"ERROR_20260715_PG_CONSTRAINT_CONTYPE_CHAR_CONCAT\",\n \"summary\": \"- Symptom: isolated Stage4H acceptance failed during schema baseline capture. - Root cause: pg_constraint.contype uses the internal PostgreSQL char type and was concatenated without an explicit text cast. - Correction: use contype::text or CAST(contype AS text). ANTI_REGRESSION_TOKEN=PG_CATALOG_CHAR_CONCAT_REQUIRES_EXPLICIT_TEXT_CAST - Machine rule: ERR-014-PG-CATALOG-CHAR-CONCAT. - Negative self-test: uncast expression rejected with RC64. - Positive self-test: explicit text cast accepted with RC0. - Task v7 mutated: no.\",\n \"rule_like\": false,\n \"signature\": \"d269f342ff6fb119a139a91810419aa2a51ca9d24dcd0797ea65318c54ef10c7\"\n },\n {\n \"id\": \"ERR-H-L925-ERROR-20260715-LIVE-COLLECTOR-VERIFIER-WRONG-PATH\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 925,\n \"title\": \"ERROR_20260715_LIVE_COLLECTOR_VERIFIER_WRONG_PATH\",\n \"summary\": \"- Symptom: verified linter installation was rolled back after the final collector integrity guard returned UNKNOWN. - Root cause: verifier queried a nonexistent path instead of the canonical live collector path declared by the task acceptance script. - Canonical path: /opt/cluster-admin-incident-engine/collector.py. - Incorrect path: /opt/homelab-cluster-admin/cluster-admin-collector.py. ANTI_REGRESSION_TOKEN=LIVE_COLLECTOR_VERIFIER_PATH_MUST_MATCH_TASK_CANONICAL_PATH - Correction: derive and guard the live collector path from the immutable task contract before executing the hash query. - Production impact: none; the canonical collector hash remained unchanged. - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": false,\n \"signature\": \"91e0cfb58ea5cec63e264f63f6ee65c13efe04ab4d4ab1f70a1d1f285f7b8ad2\"\n }\n ],\n \"rules\": [\n {\n \"id\": \"RULE-L3\",\n \"source_line\": 3,\n \"text\": \"Назначение: перед каждой следующей командой сверяться с этим файлом.\",\n \"signature\": \"910453eea5d379e1b055671660d8278df9d4565c580ecb6a036cfc1af83067f0\"\n },\n {\n \"id\": \"RULE-L11\",\n \"source_line\": 11,\n \"text\": \"## Жёсткие правила перед каждой командой\",\n \"signature\": \"3c373d80054bd99fe70a2f6e16e772fdaa67a63def211411b41f18d3123f91d0\"\n },\n {\n \"id\": \"RULE-L18\",\n \"source_line\": 18,\n \"text\": \"CHECK-7: если команда длиннее 8 строк, её нельзя давать в интерактивный shell.\",\n \"signature\": \"a9fa3d2bb79aa917672955c9adfcb227c45317ed178acb1a55b0c78505d7f10d\"\n },\n {\n \"id\": \"RULE-L36\",\n \"source_line\": 36,\n \"text\": \"Если rsync/scp упал, нельзя проверять latest offhost-файл без сверки имени.\",\n \"signature\": \"eabdf9a86e3c2ea3141f16847eb903bd516e6f12129d5e45df4f95c485794a2a\"\n },\n {\n \"id\": \"RULE-L46\",\n \"source_line\": 46,\n \"text\": \"Нельзя писать sqlite SQL вида j.type in ('object','array') внутри ssh '...'.\",\n \"signature\": \"668eb11f0a93aadac3d70e8792e586314755975dc4ac505ac13169cf55af6244\"\n },\n {\n \"id\": \"RULE-L55\",\n \"source_line\": 55,\n \"text\": \"Перед каждой командой явно понимать: где выполняется локальная часть, где remote-часть, каким пользователем.\",\n \"signature\": \"c8948237e08243f20cafec02cd56c65a19073cbef147809f7d16a7a1b027bfde\"\n },\n {\n \"id\": \"RULE-L59\",\n \"source_line\": 59,\n \"text\": \"Запрещено использовать в remote sqlite SQL фрагменты вида '$.key' или '.key'.\",\n \"signature\": \"10d38e69e3690c6892ab2aa8ddef391d5a3bdb71c9c1d80b23ebe30661d92071\"\n },\n {\n \"id\": \"RULE-L61\",\n \"source_line\": 61,\n \"text\": \"Команду с ошибкой char(36)||.dns_provider считать битой и не использовать.\",\n \"signature\": \"0ea37a4f0c68fcbd9d07b6ce4cc184b4152c5c247b1f31a1a70efa8cebeaacb9\"\n },\n {\n \"id\": \"RULE-L63\",\n \"source_line\": 63,\n \"text\": \"16. Основополагающее правило: перед каждой командой ассистент обязан явно написать: Сверка с файлом ошибок: CHECK пройден.\",\n \"signature\": \"ddd85ef9ebb1cbc0cb2d0cda3c70a5f4d23814cb7146f72672f99e2cbc8f82a0\"\n },\n {\n \"id\": \"RULE-L65\",\n \"source_line\": 65,\n \"text\": \"Нельзя выдавать команды без предварительной сверки с этим файлом ошибок.\",\n \"signature\": \"bd94bb5f40d6442f833a3ec028dfaf34c02e6d69701a7397257f9857202c2c5b\"\n },\n {\n \"id\": \"RULE-L66\",\n \"source_line\": 66,\n \"text\": \"Нельзя продолжать после собственной ошибки без записи ошибки в этот файл.\",\n \"signature\": \"f03bf4ecbe116c3281dd16b6ed1f80b16dc7a9ecd410a08244b5c41677fe5581\"\n },\n {\n \"id\": \"RULE-L68\",\n \"source_line\": 68,\n \"text\": \"19. Уточнение формата: не писать отдельную строку сверки перед каждой командой в пакете.\",\n \"signature\": \"4087c4bd5e52ca8c6f65e39b077c4bda9ae4a43aafac5857d5f33cc3f25683f0\"\n },\n {\n \"id\": \"RULE-L84\",\n \"source_line\": 84,\n \"text\": \"Не использовать больше сгенерированные большие base64 PHP для SMTP/XenForo.\",\n \"signature\": \"4db02d79ff907fa767992151c4a00ccff168749fa69417a6bcbeb6108656f2c0\"\n },\n {\n \"id\": \"RULE-L93\",\n \"source_line\": 93,\n \"text\": \"Нельзя использовать { echo RULE_CHECK_FAIL; exit 1; } в командах, выполняемых прямо в интерактивном root@pve01 shell.\",\n \"signature\": \"074e8d6400e50fd4a166ae4f67cf2da00d64051999145e17bbdaa432470ba64f\"\n },\n {\n \"id\": \"RULE-L105\",\n \"source_line\": 105,\n \"text\": \"Не использовать -crlf, если команды уже отправляются с явным \\\\r\\\\n.\",\n \"signature\": \"d8ee091ee5bb14359082cbc1e1b371d8128ddcff586b52516fed69b743f29cdf\"\n },\n {\n \"id\": \"RULE-L110\",\n \"source_line\": 110,\n \"text\": \"Не использовать heredoc Python внутри вложенного ssh; для dashboard verification использовать cat/grep JSON или простые команды.\",\n \"signature\": \"006d36b18ca6c52314d1e1d138393caf26b0d0e47533bf59ccba30210d12cbd7\"\n },\n {\n \"id\": \"RULE-L123\",\n \"source_line\": 123,\n \"text\": \"Нельзя считать monitoring слой runtime-closed только по secret scan/reference proof; нужно отдельно проверить Prometheus container/API и обновить справочник по факту.\",\n \"signature\": \"6744f09160604e98d226f32ef123dd61ee08278f0321c63edd729728300115e7\"\n },\n {\n \"id\": \"RULE-L125\",\n \"source_line\": 125,\n \"text\": \"31. Строгое правило: перед каждой командой сверяться с файлом ошибок и справочником.\",\n \"signature\": \"29acbf555ad438e9e4125ecb8c53e0ce38f6bce98ed3716e639afbb4c0eff3d7\"\n },\n {\n \"id\": \"RULE-L128\",\n \"source_line\": 128,\n \"text\": \"Если сверка не пройдена, команда не должна выполнять основное действие. Не использовать exit 1, чтобы не закрывать SSH-сессию.\",\n \"signature\": \"f20bbb59669123830e2c65160a0b67bf287cdcf82e94b17992919942fd715698\"\n },\n {\n \"id\": \"RULE-L132\",\n \"source_line\": 132,\n \"text\": \"Файл 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt нельзя считать валидным settled proof.\",\n \"signature\": \"8d28bb1cb84636890872e04ff869c76a8690936fed3b82af881bf1f2f83961e0\"\n },\n {\n \"id\": \"RULE-L133\",\n \"source_line\": 133,\n \"text\": \"Для таких проверок не использовать вложенный Python; использовать curl + grep/sed/awk либо отдельный локальный файл-скрипт.\",\n \"signature\": \"0d986ea7ea4c0cb4231812d5cf50fee6da9bdb43df2e61e2958c4536a56003b3\"\n },\n {\n \"id\": \"RULE-L176\",\n \"source_line\": 176,\n \"text\": \"- Do not split work into many small commands when the operations are low-risk, read-only, or naturally part of one evidence block.\",\n \"signature\": \"4ab34705facd281e3ea1b0cd209248f3b4087f623cf38f8b8622cbf511e67dea\"\n },\n {\n \"id\": \"RULE-L182\",\n \"source_line\": 182,\n \"text\": \"- Rule: avoid large opaque base64 script payloads for homelab changes; use transparent bounded shell/perl/python commands instead.\",\n \"signature\": \"3b1b7b83b7ed141a8dbf54ea052b6a3c6fb5701e0bc056939fa523727f996097\"\n },\n {\n \"id\": \"RULE-L185\",\n \"source_line\": 185,\n \"text\": \"- Rule: do not use large opaque payloads, generated base64 scripts, or hidden multi-line script blobs for homelab changes.\",\n \"signature\": \"386eaacb52062f19dc9f3fb9581a03ee50d5fbf08e19daba33f7271778b9b8e3\"\n },\n {\n \"id\": \"RULE-L188\",\n \"source_line\": 188,\n \"text\": \"- Any Traceback, UnicodeDecodeError, SyntaxError, Permission denied, command-not-found, nonzero critical RC, or failed write marker must force REVIEW or FAILED, never OK.\",\n \"signature\": \"540af43a414bcfd4a26cf5304ca785fbd5a127d99423647298862c964c3f493d\"\n },\n {\n \"id\": \"RULE-L196\",\n \"source_line\": 196,\n \"text\": \"- External informational widgets such as weather/Open-Meteo must not block the service launcher portal.\",\n \"signature\": \"74d434993ef9b2957b7beb9d50b3b35a30b84407a845454229c05e548bc91096\"\n },\n {\n \"id\": \"RULE-L201\",\n \"source_line\": 201,\n \"text\": \"- Rule: do not use deeply nested one-liners for config edits; first run short state checks, then apply small transparent edits.\",\n \"signature\": \"20fada2b3e3a676500b865a15816c133eb784aa10f4c6d7dab66220e573a8615\"\n },\n {\n \"id\": \"RULE-L206\",\n \"source_line\": 206,\n \"text\": \"- Matches in /logs/ or *.bak* are historical evidence only and must not make cleanup REVIEW unless the active config still contains the target.\",\n \"signature\": \"62de758aaf9d70c35a2f1fe076f80a8308e46d3b13d23c38665ab9f8ccc1cba4\"\n },\n {\n \"id\": \"RULE-L209\",\n \"source_line\": 209,\n \"text\": \"- Rule: Homepage link validation must not treat HTTP 200 as success if the body is the NPMPlus default “Congratulations” / host-not-set-up page.\",\n \"signature\": \"04b9d2a779820f3461cbb102ac47f6874198613575464488f460a9d287a089af\"\n },\n {\n \"id\": \"RULE-L217\",\n \"source_line\": 217,\n \"text\": \"- Rule: do not apply Netcraze config through stdin/multiline SSH; prove CLI input mode first.\",\n \"signature\": \"308cfe5fb5cfe99cb8c114de48913256103645e6a9e81ae9bc7ff446a484de9a\"\n },\n {\n \"id\": \"RULE-L228\",\n \"source_line\": 228,\n \"text\": \"- Rule: do not attempt ACL/config changes as routerbackup; use admin/operator router access or router Web UI for write changes.\",\n \"signature\": \"ce0e7d2dc85078db0180beacc1df838f3fe02f41db5de2d31573fe08f54a0f29\"\n },\n {\n \"id\": \"RULE-L245\",\n \"source_line\": 245,\n \"text\": \"- Rule: do not rely on custom Node lookup callback until separately tested; prefer curl --resolve or simple direct Node HTTP checks.\",\n \"signature\": \"25ad4cb5f7058222a9b5f59e98067f1db67a272e35cdd8292ca98eb050b3a0dc\"\n },\n {\n \"id\": \"RULE-L252\",\n \"source_line\": 252,\n \"text\": \"- Rule: do not keep changing ACL for this symptom; ACL is already correct. Fix service access scope or use a dedicated monitor endpoint.\",\n \"signature\": \"43f30d594ffb8d392ebc4b7defd82321e0f844a8b789a9b1d148a13df593d18c\"\n },\n {\n \"id\": \"RULE-L259\",\n \"source_line\": 259,\n \"text\": \"- Rule: do not use py_compile against root-owned system paths from an unprivileged user.\",\n \"signature\": \"ccf2aa95504d6021f9abcd8fecd6409a7a5efc61e8123fbbe7e03f539e979f89\"\n },\n {\n \"id\": \"RULE-L264\",\n \"source_line\": 264,\n \"text\": \"- Impact: do not use [PRIVATE_IP] directly as Homepage siteMonitor from edge-vm.\",\n \"signature\": \"8dfd3ac2d710c3ec6c9a2ec45f12611f948b98b8fe89d4d24b509a8d96082073\"\n },\n {\n \"id\": \"RULE-L279\",\n \"source_line\": 279,\n \"text\": \"- Rule: validate generated Python syntax locally before scp/remote execution; avoid f-strings in generated maintenance scripts.\",\n \"signature\": \"d7c5d3ed72ee048311445f823a903a5ecb6d22b03afec741f277aca10cd77858\"\n },\n {\n \"id\": \"RULE-L284\",\n \"source_line\": 284,\n \"text\": \"- Rule: do not proceed with OS baseline until SSH failure is diagnosed; likely old known_hosts key or cloud-init/root-key issue.\",\n \"signature\": \"9831f60cb061cc933e8c6688faad3f6b352d37d6237a03eefb5917a8facee420\"\n },\n {\n \"id\": \"RULE-L296\",\n \"source_line\": 296,\n \"text\": \"- Rule: avoid nested $(...) in VM SSH proofs; use literal remote commands and clean proof.\",\n \"signature\": \"65c4ac8d2adf3259f26e14c7d971b1ac68082da836bc807214901e58db59fdfa\"\n },\n {\n \"id\": \"RULE-L333\",\n \"source_line\": 333,\n \"text\": \"- Impact: do not trust that SQLite inspection attempt.\",\n \"signature\": \"23045ac8610d67d57dc1a7eb430578d6468f19d1acea9c7584f2a6f471393418\"\n },\n {\n \"id\": \"RULE-L334\",\n \"source_line\": 334,\n \"text\": \"- Rule: avoid nested SSH heredoc/Python for this task; use simpler commands, generated nginx configs, or uploaded/local script files.\",\n \"signature\": \"f4b937e830b202a02596fd60dbc1f2213645be3c5a70d929c8ff536f4adceb2c\"\n },\n {\n \"id\": \"RULE-L340\",\n \"source_line\": 340,\n \"text\": \"- Rule: read NPMplus API login values from docker inspect env internally, never print them.\",\n \"signature\": \"520fdfde2e139662a8738bb691cfe1929b9b9023a4dcc274e88a2b32ef1bfc5b\"\n },\n {\n \"id\": \"RULE-L351\",\n \"source_line\": 351,\n \"text\": \"- Impact: do not use NPMplus API for this publish path.\",\n \"signature\": \"dfce8b1c71b656add085d5c068f621a441177281533d880c8dc230b17b0e22dd\"\n },\n {\n \"id\": \"RULE-L357\",\n \"source_line\": 357,\n \"text\": \"- Impact: certbot ran without token and all certificate attempts failed; DNS cutover must not proceed from proof 535.\",\n \"signature\": \"a0e59d5dfd804bc63152339e3d2a3b0b288578d976b2039231a6272cbd9d002b\"\n },\n {\n \"id\": \"RULE-L406\",\n \"source_line\": 406,\n \"text\": \"- Rule: never source SMTP secret files containing arbitrary passwords; store password base64 and use msmtp passwordeval helper accessible to www-data.\",\n \"signature\": \"429fad764756578d3ed22029b6c2d6af0912b9b28d4a2263969cc53a090ba938\"\n },\n {\n \"id\": \"RULE-L419\",\n \"source_line\": 419,\n \"text\": \"- Rule: never store arbitrary passwords as shell-sourceable KEY=value files; use non-printing secret storage only.\",\n \"signature\": \"d04ca47a86fba60c038b212f585230b1bbb335493d583def83cbc3993bd7d88f\"\n },\n {\n \"id\": \"RULE-L424\",\n \"source_line\": 424,\n \"text\": \"- Rule: do not change DNS, firewall, proxy, mail, NetBird, or Mailcow configs before read-only triage proof.\",\n \"signature\": \"cdb23b83af44d2282df9c2a08be575953d61f280cc9abd53a4cc43b3e653cd1c\"\n },\n {\n \"id\": \"RULE-L430\",\n \"source_line\": 430,\n \"text\": \"- Rule: do not test SMTP auth again until a fresh rotated mailbox/app password is available.\",\n \"signature\": \"6694ecf8bef00c902be47d179510d8d4aadc93961c0164598e99350a41982f73\"\n },\n {\n \"id\": \"RULE-L436\",\n \"source_line\": 436,\n \"text\": \"- Impact: old timer must not be treated as valid current backup for all five forums.\",\n \"signature\": \"30e78abd678ecee3a63ddbd2d71e7caa390229722dab892fbfae0c891b725f13\"\n },\n {\n \"id\": \"RULE-L463\",\n \"source_line\": 463,\n \"text\": \"- Never print or package secrets.\",\n \"signature\": \"a0014dc66b74aee1b458771d46c51c80e582ee1bb69a1316aa6b63c91a3a40b2\"\n },\n {\n \"id\": \"RULE-L468\",\n \"source_line\": 468,\n \"text\": \"- Impact: proof 623 is invalid and must not be used to judge mail delivery.\",\n \"signature\": \"cdbba5ccdd73b604f4f3970ae1d69868a9d409f74ead6be3d60c031f26afa3f5\"\n },\n {\n \"id\": \"RULE-L497\",\n \"source_line\": 497,\n \"text\": \"- Do not print secrets.\",\n \"signature\": \"59a6b5bdc55ccb782cc5fa6d93fdf3d734c32d8250a2c21a57c6c7d4f6f3cd24\"\n },\n {\n \"id\": \"RULE-L498\",\n \"source_line\": 498,\n \"text\": \"- Do not download or upload:\",\n \"signature\": \"3de6b008bc8038faf7099e21fb98ec4280a409c05d3ae3453ec80bcd81e62240\"\n },\n {\n \"id\": \"RULE-L514\",\n \"source_line\": 514,\n \"text\": \"- Rule: do not use Cloudflare DNS-01 for these parked domains unless a token with explicit zone access is installed; because DNS already points to edge, prefer a copied-script HTTP-01 flow with temporary ACME challenge routes and content-specific validation.\",\n \"signature\": \"92d2f5a1cd4f344851af8d7a716dc24d8bc6a7738a47f6aa05013676f2200d61\"\n },\n {\n \"id\": \"RULE-L533\",\n \"source_line\": 533,\n \"text\": \"- Rule: do not run another apply stage until active NPMplus nginx route selection for newfi.ru is diagnosed with nginx -T and Host-header probes.\",\n \"signature\": \"039b358ea5679e61f5d7d8806ee36af893c18627bf8d26c434e902fb4c0e217f\"\n },\n {\n \"id\": \"RULE-L538\",\n \"source_line\": 538,\n \"text\": \"- Impact: HTTP-01 cannot work through the default server. A dedicated temporary ACME server block must be proven before certbot; do not run another certbot apply without a successful ACME-path probe for all six hostnames.\",\n \"signature\": \"2f1d3fca5be936885c2fff63166f1f43925dd8097efc48cfd46b54ff135f235a\"\n },\n {\n \"id\": \"RULE-L544\",\n \"source_line\": 544,\n \"text\": \"- Impact: do not continue certbot/apply until exact newfi.ru route behavior is isolated.\",\n \"signature\": \"8710aaa42a28a4c0a5598837af21305c7f4f17db34832d9e09cde58db0cf5a4b\"\n },\n {\n \"id\": \"RULE-L551\",\n \"source_line\": 551,\n \"text\": \"- Rule: do not use one combined parked-domain server_name block for these domains; use exact host blocks and content-specific validation.\",\n \"signature\": \"84a57025900a3fc4d1ba84dbcf54cf84b30875b82060e1e0dc66f9ccdf3da5ca\"\n },\n {\n \"id\": \"RULE-L558\",\n \"source_line\": 558,\n \"text\": \"- Rule: do not retry parked-domain HTTP-01 with low id 700-705. First prove fixed high-id exact blocks, then apply with those exact ids.\",\n \"signature\": \"527144862f28a73814ccc2c95fc4a380202a207f194962f9a7fb0a8d34833c2f\"\n },\n {\n \"id\": \"RULE-L564\",\n \"source_line\": 564,\n \"text\": \"- Rule: do not continue HTTP-01 for these parked domains; use DNS-01 with a dedicated Cloudflare token that has Zone Read and DNS Edit for the three zones.\",\n \"signature\": \"0c171db0e19ad3ff56b66bc5623f05867010cd7b2b87f98ef9cbe4d08a5c84f1\"\n },\n {\n \"id\": \"RULE-L568\",\n \"source_line\": 568,\n \"text\": \"- Impact: do not rerun Stage15 as-is.\",\n \"signature\": \"23033da11bdfd30e8d3405fa448f9e0a2516c09a778b90693eb8c01304e0b534\"\n },\n {\n \"id\": \"RULE-L569\",\n \"source_line\": 569,\n \"text\": \"- Rule: for parked domains finalization, use existing DNS-01 certificates and validate from pve01 public HTTPS with content marker; do not use the failed local --resolve TLS check as the closure gate.\",\n \"signature\": \"a6fd815206627d21af51c6ca97a5b2a03681011e1ad80eb0e6b29044cbce0323\"\n },\n {\n \"id\": \"RULE-L573\",\n \"source_line\": 573,\n \"text\": \"- Rule: Stage18 must not change DNS records or NPMplus route files. It may update placeholder HTML, run existing forum renewal script once for runtime proof, and run certbot dry-run for parked certificates using existing DNS-01 hooks.\",\n \"signature\": \"09d9b1e72b0bb7016128313662cddfa540a7a7908e768f8d4d22e5efd62168a0\"\n },\n {\n \"id\": \"RULE-L577\",\n \"source_line\": 577,\n \"text\": \"- Rule: only root gram1.ru and www.gram1.ru may be routed. Existing gram1.ru subdomain NPMplus routes must not be changed.\",\n \"signature\": \"dab2f98b8be402bac17f88d4e39ecfea8bdd5726097272cff11f7558f55eed0e\"\n },\n {\n \"id\": \"RULE-L578\",\n \"source_line\": 578,\n \"text\": \"- Rule: use DNS-01 with a dedicated gram1 Cloudflare token; do not use HTTP-01.\",\n \"signature\": \"40b057e3fd9659583584a2e3d4167a256076e134c4fbaa3678fcfc2503e55d2d\"\n },\n {\n \"id\": \"RULE-L583\",\n \"source_line\": 583,\n \"text\": \"- Decision: because pve01 has no SSH key access to the external VPS, do not modify the VPS. Instead move only pvepro.ru and www.pvepro.ru A records to edge and serve an edge landing page.\",\n \"signature\": \"17a7cd4c3204ffc9a003a8a6cef3e9be96599674d54f0f1a217eb1f3f0dad811\"\n },\n {\n \"id\": \"RULE-L584\",\n \"source_line\": 584,\n \"text\": \"- Safety rule: do not change mail.pvepro.ru, nb.pvepro.ru, MX, SPF, DKIM, DMARC or forum SMTP configuration.\",\n \"signature\": \"20b36bae3948a0b947ad12ca12a28cf8041c2ee4dd71dd71479c523fbeeaab0f\"\n },\n {\n \"id\": \"RULE-L585\",\n \"source_line\": 585,\n \"text\": \"- Use a dedicated Cloudflare token for pvepro.ru. Do not print token values.\",\n \"signature\": \"028d50fb2fef9139bce5a6a16069b7cdd69ab88719f640f0f0858034931d3d9b\"\n },\n {\n \"id\": \"RULE-L594\",\n \"source_line\": 594,\n \"text\": \"- Mistakes recorded: do not scp scripts/* after local py_compile because __pycache__ may be present; do not continue after critical scp failure in future apply stages; do not classify PVEPro landing as NetBird just because it contains a link text; use Cloudflare authoritative A records and --resolve validation when DNS propagation may lag.\",\n \"signature\": \"25354a6b9b469385d2300d5d321f8415312bb2392f2e96b37250717afce13b34\"\n },\n {\n \"id\": \"RULE-L600\",\n \"source_line\": 600,\n \"text\": \"- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet.\",\n \"signature\": \"b936ae30ffb935118585b5a133d9c32643946a3ba5034b85f42bae000518d49a\"\n },\n {\n \"id\": \"RULE-L606\",\n \"source_line\": 606,\n \"text\": \"- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet.\",\n \"signature\": \"b936ae30ffb935118585b5a133d9c32643946a3ba5034b85f42bae000518d49a\"\n },\n {\n \"id\": \"RULE-L611\",\n \"source_line\": 611,\n \"text\": \"- Rule: do not print WireGuard private keys, PSK, or client config body in proof/chat.\",\n \"signature\": \"d97ed9abb2447ca6786e0f266db8fe55e6071f854727ed9feb01a333d4fdd68e\"\n },\n {\n \"id\": \"RULE-L613\",\n \"source_line\": 613,\n \"text\": \"- Rule: do not open router public admin or certificate deployment until WireGuard handshake is proven.\",\n \"signature\": \"c44dbc849473687dad5c7756ca31b9b1366190f1f974a009ab8a2b2a175887cc\"\n },\n {\n \"id\": \"RULE-L641\",\n \"source_line\": 641,\n \"text\": \"- Do not retry immediately.\",\n \"signature\": \"e1208fd6483129c1940ee58ac88c1b14996e9906f5854c93a893423378ef3f63\"\n },\n {\n \"id\": \"RULE-L662\",\n \"source_line\": 662,\n \"text\": \"- Do not treat e3qxxx.netbird.selfhosted / 100.100.125.70 and e3qxxx-183-106.netbird.selfhosted / 100.100.183.106 as VPS egress nodes; they are user mobile peers.\",\n \"signature\": \"d025d035ce8bfa83c9f7b3605f86af415bded0a45359ee22348ce089f0a4de9f\"\n },\n {\n \"id\": \"RULE-L669\",\n \"source_line\": 669,\n \"text\": \"- Do not delete or disable Homepage siteMonitor fields to hide red badges.\",\n \"signature\": \"b101de9840659abe0fdbc458eeb21988b1ca2ef12a7cdce01e42f57991281d98\"\n },\n {\n \"id\": \"RULE-L671\",\n \"source_line\": 671,\n \"text\": \"- Do not touch Cloudflare Homepage card when operator says it is green and opens correctly.\",\n \"signature\": \"e55eaa04a75332bfdad36576da5425a956fa6e84f1003a1c606e64b0701737a5\"\n },\n {\n \"id\": \"RULE-L678\",\n \"source_line\": 678,\n \"text\": \"- Do not touch Cloudflare when operator says it is green and opens correctly.\",\n \"signature\": \"df725b9f5c7198ce4c04f9c37c1a2705147a9b2547a8f6de89800c03c68ea37f\"\n },\n {\n \"id\": \"RULE-L684\",\n \"source_line\": 684,\n \"text\": \"- Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions.\",\n \"signature\": \"b31b7d7bef8b7f89787850a3c73ce500918e256710e16626efa0d983dbec9b31\"\n },\n {\n \"id\": \"RULE-L711\",\n \"source_line\": 711,\n \"text\": \"- Future preparation must use exact privileged paths or a verified shared group, never an unprivileged wildcard after restrictive chmod.\",\n \"signature\": \"ffadddc535d9f6ef63a2e5f96282815f86c2844d21a1302af925758271cf605f\"\n },\n {\n \"id\": \"RULE-L737\",\n \"source_line\": 737,\n \"text\": \"- The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults.\",\n \"signature\": \"edb305f9a22332f3efb7192d483e3f067a1c50b93b0152596c8bbdff8b58fcd3\"\n },\n {\n \"id\": \"RULE-L768\",\n \"source_line\": 768,\n \"text\": \"- The phase-wide forbidden-pattern scan also included validate_design.py itself, which necessarily contains the prohibited literals it verifies.\",\n \"signature\": \"c9daa84dfbc8edb3cc4046148bdcb08f590a765c069f15dccba110171dcfefee\"\n },\n {\n \"id\": \"RULE-L789\",\n \"source_line\": 789,\n \"text\": \"- The backup must not be rerun merely to satisfy the invalid health-file freshness assertion.\",\n \"signature\": \"85bc7d2eeb4c251bf6db4f1e1e9be31bd05c124e10a3c3d0c9b671f1c2aea449\"\n },\n {\n \"id\": \"RULE-L793\",\n \"source_line\": 793,\n \"text\": \"- Mandatory anti-regression: после ошибки запрещено продолжать другой scope до точной root cause, исправления, повторной проверки, очистки, proof и seal.\",\n \"signature\": \"3506ad8423c9306313c4995dffee63340e95a4e1b0fdab450b433570927d3602\"\n },\n {\n \"id\": \"RULE-L818\",\n \"source_line\": 818,\n \"text\": \"- Anti-regression: never combine sudo with `< protected-file`; pass the protected path as an argument to the privileged command.\",\n \"signature\": \"8b5571b92d02dbe05c3f8a9f7637776e8665b44ce49521d4fe408fa977c13e39\"\n },\n {\n \"id\": \"RULE-L825\",\n \"source_line\": 825,\n \"text\": \"- Correction: assert required and forbidden column counts before querying recent runs.\",\n \"signature\": \"598215d2c23dd83bd6faa1beedb94f32b94ac3fd7e0573ea754db0f680b415ab\"\n },\n {\n \"id\": \"RULE-L834\",\n \"source_line\": 834,\n \"text\": \"- Anti-regression: stdin-скрипт обязан валидировать $1 по разрешённому path pattern и затем подтверждать точные SHA256 и byte count; запрещено добавлять отдельный bash-token после remote command.\",\n \"signature\": \"333aa0cc07135a79e5044a85b773ad5a7f0e655fd0a0d6422ec994230ce8112a\"\n },\n {\n \"id\": \"RULE-L867\",\n \"source_line\": 867,\n \"text\": \"- Correction: headings and prose do not contain machine-token values; validation counts only complete exact token lines.\",\n \"signature\": \"d820b99656b281c428c56ed8698a3fe4295e435e708263040de0136d1a42a18d\"\n },\n {\n \"id\": \"RULE-L869\",\n \"source_line\": 869,\n \"text\": \"- Required pattern: grep -Fxc against a complete machine line, never grep -Foc against an unrestricted substring.\",\n \"signature\": \"bab78eea9eadb2614ae16292dec8fda0bead38c2223efd2c79b512d1ad965836\"\n },\n {\n \"id\": \"RULE-L879\",\n \"source_line\": 879,\n \"text\": \"- Correction: use a linear wrapper with explicitly closed if/case blocks and avoid nested brace groups in conditional expressions.\",\n \"signature\": \"e6627533fc6f44113d3e3fb415ee4807a46afce3393d2457ca87649f3e95a338\"\n },\n {\n \"id\": \"RULE-L895\",\n \"source_line\": 895,\n \"text\": \"- Task v6 status: REJECTED_BY_LINT; never executed and never mutated.\",\n \"signature\": \"eaa651690a1e2b17124007e55346be7ad52d4d9f6172746262bc0698c376396d\"\n }\n ],\n \"command_ledger\": {\n \"present\": true,\n \"entry_count\": 10,\n \"failed_entry_count\": 0,\n \"failed_command_hashes\": []\n },\n \"privacy\": {\n \"sanitized\": true,\n \"secrets_included\": false,\n \"private_addresses_included\": false\n }\n}\n" +} +DETAIL=SECTION_STRUCTURED_ERRORS_INDEX_END +DETAIL=SECTION_WORKFLOW_CONTRACT_BEGIN +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "WORKFLOW-CONTRACT-003", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "change", + "component": "assistant-workflow", + "started_at_utc": "2026-07-21T07:41:45Z", + "finished_at_utc": "2026-07-21T07:41:45Z", + "reference_register_checked": true, + "reference_sha256": "f6f7749fda946015116cdbbd754c7e7f20356b4a6cf4df330a1b0b8fdf59cbb2", + "error_register_checked": true, + "error_register_sha256": "ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc", + "command_sha256": "d3ec6690bfb2ddbf543d9adbe2ca25d4296e193eb398776f170693ff84e91d39", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": true, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "30d935ecf9ca36e222dada4ade200a38d0066cce09656c79a60b1f25add68dc6", + "sanitized_output_sha256": "30d935ecf9ca36e222dada4ade200a38d0066cce09656c79a60b1f25add68dc6", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKFLOW_CONTRACT_VERSION=3\nCONFIRM_BACKSLASH=yes\nCONFIRM_COMMA=yes\nCONFIRM_MEANING=COMMAND_EXECUTED_READ_EXACT_IMMUTABLE_RESULT\nMANUAL_TERMINAL_OUTPUT=no\nFILE_UPLOADS=no\nCOMMAND_DISPLAY=COLLAPSED_SPOILER\nCOMMANDS_PER_STEP=1\nCOMMAND_STYLE=SHORT_TRANSPARENT\nHEREDOC_FORBIDDEN=yes\nBASE64_COMMANDS_FORBIDDEN=yes\nLARGE_INLINE_COMMANDS_FORBIDDEN=yes\nRESULT_SOURCE=IMMUTABLE_HISTORY_EXACT_COMMAND_ID\nLATEST_JSON_AUTHORITATIVE=no\nBEFORE_COMMAND=LOAD_BOOTSTRAP_ERRORS_INDEX_PREVIOUS_RESULT\nAFTER_FAILURE=READ_PUBLISHED_FAILURE_BEFORE_NEXT_COMMAND\nNO_REPEAT_EXACT_FAILED_COMMAND=yes\nMISSING_RESULT_CHECK=EXACT_FILE_THEN_HISTORY_DIRECTORY_THEN_BRANCH_HEAD\nNEW_CHAT_INPUT=BOOTSTRAP_URL_ONLY\n" +} +DETAIL=SECTION_WORKFLOW_CONTRACT_END +DETAIL=SECTION_DOUBLE_CHECK_RULE_BEGIN +{"schema_version":1,"command_id":"DOUBLE-CHECK-RULE-001","status":"OK","strict":true,"rule":"two_independent_pre_command_self_checks","checks":["bootstrap_errors_previous_result_failed_hash","syntax_quoting_target_permissions_side_effects_rollback_success_criteria"]} +DETAIL=SECTION_DOUBLE_CHECK_RULE_END +DETAIL=SECTION_FRESH_AUTO_CONTEXT_BEGIN +CHAT_OUTPUT_BEGIN +COMMAND_ID=CONTEXT-AUTO-[PRIVATE_IP]652Z +STATUS=OK +RC=0 +HOST=pve01 +COMPONENT=cluster-context +REFERENCE_SHA[PRIVATE_IP]f8edc75fcf[PRIVATE_IP]c[PRIVATE_IP]cc112ceccb6cf[PRIVATE_IP]a4deb8e +ERROR_REGISTER_SHA[PRIVATE_IP]b8e38eaa1acec915352fb[PRIVATE_IP]caa7ff6cd9d1ec0da[PRIVATE_IP] +OUTPUT_BEGIN +GENERATED_AT_UTC=[PRIVATE_IP]T14:16:52Z +Cluster information +------------------- +Name: homelab +Config Version: 3 +Transport: knet +Secure auth: on + +Quorum information +------------------ +Date: Thu Jul [PRIVATE_IP] 2026 +Quorum provider: corosync_votequorum +Nodes: 3 +Node ID: [PRIVATE_IP] +Ring ID: 1.ba +Quorate: Yes + +Votequorum information +---------------------- +Expected votes: 3 +Highest expected: 3 +Total votes: 3 +Quorum: 2 +Flags: Quorate + +Membership information +---------------------- + Nodeid Votes Name +[PRIVATE_IP] [PRIVATE_IP].11 (local) +[PRIVATE_IP] [PRIVATE_IP].13 +[PRIVATE_IP] [PRIVATE_IP].12 + +Membership information +---------------------- + Nodeid Votes Name + 1 1 pve01 (local) + 2 1 pve03 + 3 1 pve02 +[{"cpu":[PRIVATE_IP]801e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/110","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns1","netin":[PRIVATE_IP],"netout":641185,"node":"pve01","status":"running","template":0,"type":"lxc","uptime":3345,"vmid":110},{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/111","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns2","netin":[PRIVATE_IP],"netout":272209,"node":"pve02","status":"running","template":0,"type":"lxc","uptime":3178,"vmid":111},{"cpu":[PRIVATE_IP]801e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/112","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound1","netin":[PRIVATE_IP],"netout":780728,"node":"pve01","status":"running","template":0,"type":"lxc","uptime":3348,"vmid":112},{"cpu":[PRIVATE_IP]424e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/113","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound2","netin":[PRIVATE_IP],"netout":184449,"node":"pve02","status":"running","template":0,"type":"lxc","uptime":3181,"vmid":113},{"cpu":[PRIVATE_IP]20667,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"edge-vm","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":3286,"vmid":130},{"cpu":[PRIVATE_IP]49185,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/150","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"nextcloud","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":3343,"vmid":150},{"cpu":[PRIVATE_IP]98795,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/160","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"forum-prod","netin":[PRIVATE_IP],"netout":65626,"node":"pve02","status":"running","template":0,"type":"qemu","uptime":3175,"vmid":160},{"cpu":[PRIVATE_IP]05248,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/170","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"core-apps","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":3339,"vmid":170},{"cpu":[PRIVATE_IP]91491,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/171","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"monitoring","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":3336,"vmid":171},{"cpu":[PRIVATE_IP]31022,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/180","maxcpu":2,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"cluster-admin","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"qemu","uptime":3172,"vmid":180},{"cpu":[PRIVATE_IP]87303,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/190","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"homelab-ops-worker","netin":[PRIVATE_IP],"netout":194110,"node":"pve03","status":"running","template":0,"type":"qemu","uptime":3282,"vmid":190},{"cpu":0,"disk":0,"diskread":0,"diskwrite":0,"id":"qemu/9130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":0,"memhost":0,"name":"edge-cold-standby","netin":0,"netout":0,"node":"pve02","status":"stopped","template":0,"type":"qemu","uptime":0,"vmid":9130},{"cgroup-mode":2,"cpu":[PRIVATE_IP]24746,"disk":[PRIVATE_IP],"id":"node/pve02","level":"","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve02","status":"online","type":"node","uptime":3242},{"cgroup-mode":2,"cpu":[PRIVATE_IP]96147,"disk":[PRIVATE_IP],"id":"node/pve03","level":"","maxcpu":8,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve03","status":"online","type":"node","uptime":3303},{"cgroup-mode":2,"cpu":[PRIVATE_IP]41631,"disk":[PRIVATE_IP],"id":"node/pve01","level":"","maxcpu":24,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve01","status":"online","type":"node","uptime":3376},{"content":"iso,import,vztmpl,backup","disk":[PRIVATE_IP],"id":"storage/pve02/local","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"iso,import,vztmpl,backup","disk":[PRIVATE_IP],"id":"storage/pve03/local","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"iso,import,vztmpl,backup","disk":[PRIVATE_IP],"id":"storage/pve01/local","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve02/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve03/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve01/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"id":"network/pve02/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve02","status":"ok","type":"network"},{"id":"network/pve03/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve03","status":"ok","type":"network"},{"id":"network/pve01/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve01","status":"ok","type":"network"}] +Name Type Status Total (KiB) Used (KiB) Available (KiB) % +local dir active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 41.13% +local-lvm lvmthin active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 38.36% + UNIT LOAD ACTIVE SUB DESCRIPTION +● homelab-crypto-portfolio-chart-health.service loaded failed failed Crypto portfolio chart health check +● skladchik-reports-monitor-supervisor.service loaded failed failed Skladchik reports monitor full supervisor + +Legend: LOAD → Reflects whether the unit definition was properly loaded. + ACTIVE → The high-level unit activation state, i.e. generalization of SUB. + SUB → The low-level unit activation state, values depend on unit type. + +2 loaded units listed. +NEXT LEFT LAST PASSED UNIT ACTIVATES +Thu [PRIVATE_IP] 17:17:11 MSK 18s Thu [PRIVATE_IP] 17:16:07 MSK 45s ago homelab-router-watchdog.timer homelab-router-watchdog.service +Thu [PRIVATE_IP] 17:17:15 MSK 22s Thu [PRIVATE_IP] 17:12:15 MSK 4min 37s ago homelab-smartctl-textfile.timer homelab-smartctl-textfile.service +Thu [PRIVATE_IP] 17:17:19 MSK 26s Thu [PRIVATE_IP] 17:12:08 MSK 4min 44s ago homelab-health-metrics.timer homelab-health-metrics.service +Thu [PRIVATE_IP] 17:17:28 MSK 35s Thu [PRIVATE_IP] 17:16:28 MSK 24s ago homelab-cluster-admin-webpanel-sync.timer homelab-cluster-admin-webpanel-sync.service +Thu [PRIVATE_IP] 17:17:28 MSK 35s Thu [PRIVATE_IP] 17:16:28 MSK 24s ago homelab-private-vpn-hosts-health.timer homelab-private-vpn-hosts-health.service +Thu [PRIVATE_IP] 17:17:32 MSK 39s Thu [PRIVATE_IP] 17:12:08 MSK 4min 44s ago homelab-crypto-portfolio-chart-health.timer homelab-crypto-portfolio-chart-health.service +Thu [PRIVATE_IP] 17:18:28 MSK 1min 35s Thu [PRIVATE_IP] 17:13:28 MSK 3min 24s ago netbird-peers-health.timer netbird-peers-health.service +Thu [PRIVATE_IP] 17:20:58 MSK 4min 5s Thu [PRIVATE_IP] 17:05:58 MSK 10min ago prometheus-node-exporter-apt.timer prometheus-node-exporter-apt.service +Thu [PRIVATE_IP] 17:20:58 MSK 4min 5s Thu [PRIVATE_IP] 17:05:58 MSK 10min ago prometheus-node-exporter-nvme.timer prometheus-node-exporter-nvme.service +Thu [PRIVATE_IP] 17:23:15 MSK 6min Thu [PRIVATE_IP] 17:08:15 MSK 8min ago homelab-disk-space-health.timer homelab-disk-space-health.service +Thu [PRIVATE_IP] 17:23:15 MSK 6min Thu [PRIVATE_IP] 17:08:15 MSK 8min ago homelab-pve03-staging-capacity-health.timer homelab-pve03-staging-capacity-health.service +Thu [PRIVATE_IP] 17:26:04 MSK 9min Thu [PRIVATE_IP] 17:16:04 MSK 48s ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service +Thu [PRIVATE_IP] 17:30:00 MSK 13min Thu [PRIVATE_IP] 17:15:04 MSK 1min 48s ago homelab-incident-journal.timer homelab-incident-journal.service +Thu [PRIVATE_IP] 17:30:00 MSK 13min Thu [PRIVATE_IP] 17:15:04 MSK 1min 48s ago homelab-safe-autoheal.timer homelab-safe-autoheal.service +Thu [PRIVATE_IP] 17:30:14 MSK 13min Thu [PRIVATE_IP] 17:16:09 MSK 43s ago homelab-duty-admin-v2.timer homelab-duty-admin-v2.service +Thu [PRIVATE_IP] 17:30:30 MSK 13min Thu [PRIVATE_IP] 17:16:04 MSK 48s ago homelab-external-probe-vps-health.timer homelab-external-probe-vps-health.service +Thu [PRIVATE_IP] 17:32:05 MSK 15min Thu [PRIVATE_IP] 17:04:00 MSK 12min ago homelab-mkdocs-auto-refresh.timer homelab-mkdocs-auto-refresh.service +Thu [PRIVATE_IP] 17:39:47 MSK 22min Thu [PRIVATE_IP] 17:08:37 MSK 8min ago homelab-reference-refresh.timer homelab-reference-refresh.service +Thu [PRIVATE_IP] 21:43:01 MSK 4h 26min Wed [PRIVATE_IP] 23:00:09 MSK - apt-daily.timer apt-daily.service +Thu [PRIVATE_IP] 22:23:37 MSK 5h 6min Thu [PRIVATE_IP] 16:23:37 MSK 53min ago homelab-vps-identity-audit.timer homelab-vps-identity-audit.service +Fri [PRIVATE_IP] 00:00:00 MSK 6h Thu [PRIVATE_IP] 00:00:07 MSK - dpkg-db-backup.timer dpkg-db-backup.service +Fri [PRIVATE_IP] 00:03:01 MSK 6h Thu [PRIVATE_IP] 00:03:53 MSK - homelab-evidence-catalog.timer homelab-evidence-catalog.service +Fri [PRIVATE_IP] 00:03:32 MSK 6h Thu [PRIVATE_IP] 00:10:41 MSK - homelab-quality-gate.timer homelab-quality-gate.service +Fri [PRIVATE_IP] 00:05:00 MSK 6h Thu [PRIVATE_IP] 00:03:41 MSK - homelab-backup-freshness.timer homelab-backup-freshness.service +Fri [PRIVATE_IP] 00:19:19 MSK 7h Thu [PRIVATE_IP] 00:14:04 MSK - homelab-storage-capacity.timer homelab-storage-capacity.service +Fri [PRIVATE_IP] 00:19:49 MSK 7h Thu [PRIVATE_IP] 00:09:41 MSK - homelab-docker-health.timer homelab-docker-health.service +Fri [PRIVATE_IP] 00:51:58 MSK 7h Thu [PRIVATE_IP] 00:52:07 MSK - logrotate.timer logrotate.service +Fri [PRIVATE_IP] 01:38:37 MSK 8h Thu [PRIVATE_IP] 04:54:14 MSK - pve-daily-update.timer pve-daily-update.service +Fri [PRIVATE_IP] 02:15:00 MSK 8h Thu [PRIVATE_IP] 02:15:14 MSK - homelab-vm-full-backup.timer homelab-vm-full-backup.service +Fri [PRIVATE_IP] 03:13:22 MSK 9h Thu [PRIVATE_IP] 03:14:25 MSK - homelab-emergency-backup.timer homelab-emergency-backup.service +Fri [PRIVATE_IP] 03:35:00 MSK 10h Thu [PRIVATE_IP] 03:35:07 MSK - filebrowser-backup.timer filebrowser-backup.service +Fri [PRIVATE_IP] 03:48:11 MSK 10h Thu [PRIVATE_IP] 03:53:25 MSK - homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service +Fri [PRIVATE_IP] 03:50:00 MSK 10h Thu [PRIVATE_IP] 03:50:07 MSK - filebrowser-offhost-sync.timer filebrowser-offhost-sync.service +Fri [PRIVATE_IP] 04:10:00 MSK 10h Thu [PRIVATE_IP] 04:10:14 MSK - filebrowser-restore-validate.timer filebrowser-restore-validate.service +Fri [PRIVATE_IP] 04:17:17 MSK 11h Thu [PRIVATE_IP] 04:06:57 MSK - homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service +Fri [PRIVATE_IP] 04:27:28 MSK 11h Thu [PRIVATE_IP] 04:30:14 MSK - homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service +Fri [PRIVATE_IP] 04:28:06 MSK 11h Thu [PRIVATE_IP] 04:32:14 MSK - skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service +Fri [PRIVATE_IP] 04:46:18 MSK 11h Thu [PRIVATE_IP] 05:05:14 MSK - homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service +Fri [PRIVATE_IP] 04:49:38 MSK 11h Thu [PRIVATE_IP] 04:39:08 MSK - homelab-edge-vm-offhost-freshness.timer homelab-edge-vm-offhost-freshness.service +Fri [PRIVATE_IP] 05:12:27 MSK 11h Thu [PRIVATE_IP] 05:03:14 MSK - homelab-sops-edge-secret-coverage.timer homelab-sops-edge-secret-coverage.service +Fri [PRIVATE_IP] 05:15:18 MSK 11h Thu [PRIVATE_IP] 05:23:15 MSK - homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service +Fri [PRIVATE_IP] 05:17:49 MSK 12h Thu [PRIVATE_IP] 05:30:14 MSK - homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service +Fri [PRIVATE_IP] 05:54:13 MSK 12h Thu [PRIVATE_IP] 05:41:49 MSK - homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service +Fri [PRIVATE_IP] 05:58:44 MSK 12h Thu [PRIVATE_IP] 06:13:25 MSK - homelab-extended-appbackup.timer homelab-extended-appbackup.service +Fri [PRIVATE_IP] 06:34:41 MSK 13h Thu [PRIVATE_IP] 06:23:41 MSK - apt-daily-upgrade.timer apt-daily-upgrade.service +Fri [PRIVATE_IP] 06:35:45 MSK 13h Thu [PRIVATE_IP] 06:32:14 MSK - homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service +Fri [PRIVATE_IP] 06:39:45 MSK 13h Thu [PRIVATE_IP] 06:38:30 MSK - homelab-alertmanager-backup.timer homelab-alertmanager-backup.service +Fri [PRIVATE_IP] 06:41:37 MSK 13h Thu [PRIVATE_IP] 06:54:41 MSK - homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service +Fri [PRIVATE_IP] 06:45:23 MSK 13h Thu [PRIVATE_IP] 06:41:07 MSK - homelab-verified-backup-health.timer homelab-verified-backup-health.service +Fri [PRIVATE_IP] 06:50:23 MSK 13h Thu [PRIVATE_IP] 06:48:14 MSK - homelab-beszel-backup.timer homelab-beszel-backup.service +Fri [PRIVATE_IP] 07:04:26 MSK 13h Thu [PRIVATE_IP] 07:08:14 MSK - homelab-backup-healthcheck.timer homelab-backup-healthcheck.service +Fri [PRIVATE_IP] 07:04:54 MSK 13h Thu [PRIVATE_IP] 07:01:31 MSK - homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service +Fri [PRIVATE_IP] 07:10:46 MSK 13h Thu [PRIVATE_IP] 07:06:41 MSK - homelab-drift-check.timer homelab-drift-check.service +Fri [PRIVATE_IP] 07:15:30 MSK 13h Thu [PRIVATE_IP] 07:12:25 MSK - homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service +Fri [PRIVATE_IP] 07:23:47 MSK 14h Thu [PRIVATE_IP] 07:30:14 MSK - homelab-service-registry-check.timer homelab-service-registry-check.service +Fri [PRIVATE_IP] 07:27:41 MSK 14h Thu [PRIVATE_IP] 07:28:41 MSK - homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service +Fri [PRIVATE_IP] 07:41:37 MSK 14h Thu [PRIVATE_IP] 07:53:14 MSK - homelab-dependency-map-check.timer homelab-dependency-map-check.service +Fri [PRIVATE_IP] 07:42:38 MSK 14h Thu [PRIVATE_IP] 07:49:17 MSK - homelab-vm-backup-health.timer homelab-vm-backup-health.service +Fri [PRIVATE_IP] 07:46:04 MSK 14h Thu [PRIVATE_IP] 07:53:30 MSK - homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service +Fri [PRIVATE_IP] 07:46:57 MSK 14h Thu [PRIVATE_IP] 07:52:25 MSK - man-db.timer man-db.service +Fri [PRIVATE_IP] 07:48:15 MSK 14h Thu [PRIVATE_IP] 07:37:25 MSK - homelab-alerting-health.timer homelab-alerting-health.service +Fri [PRIVATE_IP] 07:57:47 MSK 14h Thu [PRIVATE_IP] 07:46:34 MSK - homelab-runbook-generate.timer homelab-runbook-generate.service +Fri [PRIVATE_IP] 08:01:16 MSK 14h Thu [PRIVATE_IP] 07:56:14 MSK - homelab-duty-admin-report.timer homelab-duty-admin-report.service +Fri [PRIVATE_IP] 08:05:15 MSK 14h Thu [PRIVATE_IP] 08:06:41 MSK - homelab-desired-state-sync.timer homelab-desired-state-sync.service +Fri [PRIVATE_IP] 08:09:51 MSK 14h Thu [PRIVATE_IP] 08:03:14 MSK - homelab-overall-health.timer homelab-overall-health.service +Fri [PRIVATE_IP] 08:11:13 MSK 14h Thu [PRIVATE_IP] 08:14:17 MSK - homelab-kuma-monitor-policy.timer homelab-kuma-monitor-policy.service +Fri [PRIVATE_IP] 08:19:51 MSK 15h Thu [PRIVATE_IP] 08:06:25 MSK - homelab-backup-sla-health.timer homelab-backup-sla-health.service +Fri [PRIVATE_IP] 08:25:29 MSK 15h Thu [PRIVATE_IP] 08:15:14 MSK - homelab-final-readiness-gate.timer homelab-final-readiness-gate.service +Fri [PRIVATE_IP] 08:26:16 MSK 15h Thu [PRIVATE_IP] 08:29:47 MSK - homelab-vm-backup-policy.timer homelab-vm-backup-policy.service +Fri [PRIVATE_IP] 08:34:22 MSK 15h Thu [PRIVATE_IP] 08:24:07 MSK - homelab-capacity-risk.timer homelab-capacity-risk.service +Fri [PRIVATE_IP] 08:35:10 MSK 15h Thu [PRIVATE_IP] 08:47:05 MSK - memos-backup.timer memos-backup.service +Fri [PRIVATE_IP] 08:37:40 MSK 15h Thu [PRIVATE_IP] 08:47:25 MSK - homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service +Fri [PRIVATE_IP] 08:44:49 MSK 15h Thu [PRIVATE_IP] 08:54:14 MSK - homelab-cluster-passport.timer homelab-cluster-passport.service +Fri [PRIVATE_IP] 08:46:04 MSK 15h Thu [PRIVATE_IP] 08:37:14 MSK - homelab-secret-exposure-guard.timer homelab-secret-exposure-guard.service +Fri [PRIVATE_IP] 08:54:27 MSK 15h Thu [PRIVATE_IP] 08:53:30 MSK - immich-media-offhost-sync.timer immich-media-offhost-sync.service +Fri [PRIVATE_IP] 08:56:22 MSK 15h Thu [PRIVATE_IP] 09:00:41 MSK - homelab-golden-state-index.timer homelab-golden-state-index.service +Fri [PRIVATE_IP] 09:11:08 MSK 15h Thu [PRIVATE_IP] 09:24:25 MSK - memos-offhost-sync.timer memos-offhost-sync.service +Fri [PRIVATE_IP] 10:05:51 MSK 16h Thu [PRIVATE_IP] 10:12:14 MSK - paperless-restore-dry-run.timer paperless-restore-dry-run.service +Fri [PRIVATE_IP] 10:15:00 MSK 16h Thu [PRIVATE_IP] 10:15:13 MSK - skladchik-reports-monitor-monthly-selftest-watch.timer skladchik-reports-monitor-monthly-selftest-watch.service +Fri [PRIVATE_IP] 16:35:37 MSK 23h Thu [PRIVATE_IP] 16:35:37 MSK 41min ago systemd-tmpfiles-clean.timer systemd-tmpfiles-clean.service +Sun [PRIVATE_IP] 03:10:03 MSK 2 days Sun [PRIVATE_IP] 03:10:29 MSK - e2scrub_all.timer e2scrub_all.service +Sun [PRIVATE_IP] 03:10:35 MSK 2 days Sun [PRIVATE_IP] 03:10:14 MSK - xfs_scrub_all.timer xfs_scrub_all.service +Sun [PRIVATE_IP] 03:34:48 MSK 2 days Sun [PRIVATE_IP] 03:45:37 MSK - homelab-auto-backup.timer homelab-auto-backup.service +Sun [PRIVATE_IP] 05:15:13 MSK 2 days Sun [PRIVATE_IP] 05:11:25 MSK - skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service +Sun [PRIVATE_IP] 09:39:32 MSK 2 days Sun [PRIVATE_IP] 09:32:29 MSK - homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service +Sun [PRIVATE_IP] 10:11:29 MSK 2 days Sun [PRIVATE_IP] 10:06:07 MSK - homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service +Sun [PRIVATE_IP] 11:28:24 MSK 2 days Sun [PRIVATE_IP] 11:08:41 MSK - homelab-vm-local-dumps-retention.timer homelab-vm-local-dumps-retention.service +Mon [PRIVATE_IP] 00:07:15 MSK 3 days Mon [PRIVATE_IP] 00:00:29 MSK - homelab-secret-sanity.timer homelab-secret-sanity.service +Mon [PRIVATE_IP] 01:33:11 MSK 3 days Mon [PRIVATE_IP] 00:51:14 MSK - fstrim.timer fstrim.service +Mon [PRIVATE_IP] 07:34:52 MSK 3 days Mon [PRIVATE_IP] 07:25:14 MSK - homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service +Thu [PRIVATE_IP] 06:00:00 MSK 2 months 22 days Wed [PRIVATE_IP] 06:00:00 MSK - homelab-vm-full-restore-drill.timer homelab-vm-full-restore-drill.service +- - Thu [PRIVATE_IP] 17:16:53 MSK 17ms ago homelab-cluster-admin-inventory-sync.timer homelab-cluster-admin-inventory-sync.service +- - Thu [PRIVATE_IP] 17:16:04 MSK 48s ago homelab-forum-snuffleupagus-health.timer homelab-forum-snuffleupagus-health.service +- - - - prometheus-node-exporter-ipmitool-sensor.timer prometheus-node-exporter-ipmitool-sensor.service +- - - - prometheus-node-exporter-mellanox-hca-temp.timer prometheus-node-exporter-mellanox-hca-temp.service +- - - - prometheus-node-exporter-smartmon.timer prometheus-node-exporter-smartmon.service + +96 timers listed. +- Обрезанная, повреждённая или частично вставленная команда считается невыполненной; сначала проводится side-effect audit. +- Хвост parse-ошибки закрывается только после доказательства отсутствия изменений и временных артефактов. +- Proof: /var/lib/homelab-change-sets/operator-command-size-guard-v[PRIVATE_IP]T182653Z/report.txt + +## HOMELAB_ADMIN_CLI_CONTRACT_[PRIVATE_IP] +- Поддерживаемые команды: version, status, lint TASK_DIR, validate TASK_DIR, run TASK_DIR PHASE. +- Отдельная команда --help отсутствует. Неизвестный аргумент печатает usage и штатно возвращает RC=64. +- Нельзя трактовать RC=64 от --help как неисправность runner. +- Для проверки доступности runner использовать homelab-admin version; для структуры CLI использовать зафиксированную usage-строку. +- Proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v[PRIVATE_IP]T183338Z/report.txt + +AUTOMATIC_CLUSTER_CONTEXT_REFRESH_[PRIVATE_IP] +Collector: /usr/local/sbin/homelab-context-collect +Scheduler: hourly at minute 17 via /etc/cron.d/homelab-context-refresh +Destination: immutable Gitea runtime history with CONTEXT-AUTO UTC command IDs. + +AUTOMATIC_CLUSTER_CONTEXT_REFRESH_DIRECT_[PRIVATE_IP] +Publisher: /usr/local/sbin/homelab-context-refresh-direct +Schedule: hourly at minute 17. +Destination: immutable assistant-runtime history with CONTEXT-AUTO UTC IDs. +- Registered at: [PRIVATE_IP]T06:27:06Z + +AUTOREFRESH_INLINE_FAILURE_[PRIVATE_IP] +Attempts 048 and 049 did not appear in immutable history. +Rule: use validated local helper scripts and a simple scheduler entry; never embed the full collector in cron or nested SSH quoting. + +AUTOREFRESH_WRAPPER_PUBLICATION_FAILURE_[PRIVATE_IP] +Commands [PRIVATE_IP] did not reach immutable history through homelab-chat-run. +Resolution: hourly snapshots use direct scp plus homelab-runtime-receive publication. + +AUTOREFRESH_052_NOT_PUBLISHED_[PRIVATE_IP] +CONTEXT-AUTOREFRESH-ACTIVATE-052 did not appear in immutable history; direct publisher retry requires sanitization before transmission. + +IMMUTABLE_HISTORY_FALSE_NEGATIVE_[PRIVATE_IP] +BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe[PRIVATE_IP]d4a62bf[PRIVATE_IP]bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative. +Rule: verify the exact COMMAND_ID result, wrapper publication status and commit SHA; never infer absence from a cached directory listing. + +BACKUP_UNSAFE_SOURCE_CAPTURE_071_PARSE_ERROR_[PRIVATE_IP] +BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (. +Resolution: avoid nested quote layers; parse targets first and pass file paths as simple SSH arguments. +OUTPUT_END +CHAT_OUTPUT_END +DETAIL=SECTION_FRESH_AUTO_CONTEXT_END +DETAIL=FULL_CONTEXT_BUNDLE_END +DETAIL=CONTEXT_SOURCES_FULL_REFRESH=PASS +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 6ad82d8..9f7d665 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1 +1,30 @@ -{"schema_version":1,"channel":"homelab-runtime","command_id":"CONTEXT-AUTO-20260723T141701Z","status":"OK","rc":0,"host":"pve01","mode":"read-only","component":"cluster-context","started_at_utc":"2026-07-23T14:17:01Z","finished_at_utc":"2026-07-23T14:17:02Z","reference_register_checked":true,"reference_sha256":"5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e","error_register_checked":true,"error_register_sha256":"08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752","changes_made":false,"sanitized":true,"secrets_included":false,"private_addresses_included":false,"output":"AUTOMATIC_CLUSTER_CONTEXT_REFRESH=OK"} +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "CONTEXT-SOURCES-FULL-REFRESH-363", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-context", + "started_at_utc": "2026-07-23T14:23:43Z", + "finished_at_utc": "2026-07-23T14:23:43Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "eb33834ec5df95efdbe55c4264ce02cbc69fcf90de62d457948a23d1e1efda04", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "9482ef04e0765ed01d6de068ebba4ef30fb0e133871b6cd45c4af4906d299122", + "sanitized_output_sha256": "3d9b0a662d873becba40d548b6854862d5ad518285238a73362573ff85988751", + "output_truncated_in_json": true, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/test\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/mktemp\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/rm\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/curl\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/jq\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/stat\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/sha256sum\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/awk\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/date\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/cat\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/printf\nDETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/grep\nDETAIL=STAGE=SOURCE_FETCH\nDETAIL=IMMUTABLE_SOURCE=errors-index|SIZE=146628|SHA256=a13ca59d5fa1ad39b813bad3115d7b8cf8c5c68935fc813dbd792757ed421e24|STATUS=PASS\nDETAIL=IMMUTABLE_SOURCE=workflow-contract|SIZE=1926|SHA256=1378a2c0d6dec6b09e3ea7dfced65fed1e1c38065baa6bba8bfd4c89e23e2c78|STATUS=PASS\nDETAIL=IMMUTABLE_SOURCE=double-check-rule|SIZE=268|SHA256=7e5221154959bb0f435b3cfb955e6671d3c361ecc7a99625f04252cc74ebaa4b|STATUS=PASS\nDETAIL=FULL_CONTEXT_BUNDLE_BEGIN\nFULL_CONTEXT_BUNDLE_SCHEMA=1\nGENERATED_AT_UTC=2026-07-23T14:23:43Z\nREFERENCE_FILE=/etc/pve/31_HOMELAB_REFERENCE.md\nREFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e\nREFERENCE_SIZE=164602\nERROR_REGISTER_FILE=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\nERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752\nERROR_REGISTER_SIZE=79246\nAUTO_CONTEXT_ID=CONTEXT-AUTO-20260723T141652Z\nAUTO_CONTEXT_COMMIT=fcf2ed2a6b11016ec6441a014f7832a86c51f793\nAUTO_CONTEXT_URL=https://git.gram1.ru/homelab-admin/homelab-public-context/raw/commit/fcf2ed2a6b11016ec6441a014f7832a86c51f793/runtime/history/CONTEXT-AUTO-20260723T141652Z.txt\nAUTO_CONTEXT_SHA256=acd6a206ec5eb45ed50c2aae1cee02d4683dd8675e5c2d88190cefcd2446dada\nAUTO_CONTEXT_SIZE=29049\nLATEST_JSON_AUTHORITATIVE=NO\nDETAIL=SECTION_REFERENCE_REGISTER_BEGIN\nHOMELAB REFERENCE\nGENERATED=2026-06-29T21:45:29+03:00\nAUDIT_DIR=/root/cluster-audit-20260629T201245\n\nCORE_CLUSTER_CONFIG\n\nkeyboard: en-us\nmigration: secure,network=[PRIVATE_IP]/24\n\nFINAL_CLASSIFICATION\nMISSING_EXPECTED_PREFIX_COUNT=0\nSTRICT_POSSIBLE_SECRET_COUNT=0\nSTRICT_SECRET_SCAN_OK\nHEALTH_WARN_ERROR_COUNT=14\nNOTE exclude_from_reference_or_delete_later=05_edge_compose_safe.tgz\nNOTE forum-prod VMID160 is not in homelab-nightly-all; it has separate local/cloud backup chain\nNOTE edge-vm disk scsi1 backup=0 risk must be documented\nNOTE Corosync remains on [PRIVATE_IP]/12/13; migration network is [PRIVATE_IP]/24 secure\nNOTE edge health WARN/ERROR items should be documented as known current states\n\nHEALTH_NOT_OK\nHEALTH_NOT_OK item=adguard-rewrite-sync.txt line=STATUS=ERROR TS=20260629T152553Z TYPE=adguard-rewrite-sync REASON=URLError SECRET_PRINTED=REDACTED\nHEALTH_NOT_OK item=backup-restore-coverage-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=backup-restore-dashboard.txt line=STATUS=WARN TS=2026-06-29T18:10:09Z ITEMS=80 NOT_OK=13\nHEALTH_NOT_OK item=container-image-lifecycle-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=cron-job-monitoring-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=direct-heartbeat-pilot-readiness.txt line=STATUS=WARN\nHEALTH_NOT_OK item=healthchecks-heartbeat-coverage.txt line=STATUS=WARN\nHEALTH_NOT_OK item=healthchecks-job-coverage.txt line=STATUS=WARN\nHEALTH_NOT_OK item=healthchecks-job-monitoring-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=homepage-service-catalog-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=ingress-dns-route-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=job-monitoring-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=observability-health-surface-slo.txt line=STATUS=WARN\nHEALTH_NOT_OK item=security-vulnerability-slo.txt line=STATUS=WARN\n\nEVIDENCE_FILES\n00_cluster_overview.txt 3544 bytes\n00_HOMELAB_ASSISTANT_ERROR_REGISTER.md 2566 bytes\n01_pve01_audit.txt 28554 bytes\n02_pve02_audit.txt 24786 bytes\n03_pve03_audit.txt 16842 bytes\n04_edge_vm_basic.txt 34506 bytes\n05_edge_compose_redacted.txt 24228 bytes\n05_edge_compose_safe.tgz.QUARANTINED.txt 354 bytes\n05_edge_compose_tar_errors.txt 166 bytes\n06_ACCESS_AND_ERROR_RULES.md 13742 bytes\n06_edge_npmplus_routes_safe.txt 17350 bytes\n07_edge_systemd_backup_audit.txt 20694 bytes\n08_edge_scripts_redacted.txt 198622 bytes\n09_forum_prod_basic.txt 14413 bytes\n10_forum_codevipe_xenforo_audit.txt 3861 bytes\n11_forum_backup_audit_redacted.txt 5952 bytes\n12_pve02_codevipe_cloud_backup_audit.txt 6277 bytes\n13_pve01_systemd_backup_audit.txt 60109 bytes\n14_pve01_scripts_redacted.txt 246474 bytes\n15_pve01_ct_110_audit.txt 5049 bytes\n15_pve01_ct_112_audit.txt 4843 bytes\n16_pve02_ct_111_audit.txt 4758 bytes\n16_pve02_ct_113_audit.txt 4843 bytes\n17_nextcloud_vm_audit.txt 13293 bytes\n18_nextcloud_backup_restore_pve01_audit.txt 4269 bytes\n19_pve02_host_automation_audit.txt 10642 bytes\n19_pve03_host_automation_audit.txt 7632 bytes\n20_AUDIT_INDEX_AND_SECRET_SCAN.txt 11935 bytes\n21_proxmox_cluster_config_audit.txt 10855 bytes\n22_internal_2_5g_network_inventory.txt 8036 bytes\n23_cluster_internal_network_configured.txt 747 bytes\n24_cluster_internal_network_speedtest.txt 54665 bytes\n25_cluster_internal_network_migration_enabled.txt 1816 bytes\n26_migration_network_pvesh_verify.txt 1210 bytes\n27_migration_network_canonical_verify.txt 956 bytes\n28_dns_configs_redacted.txt 15559 bytes\n29_health_summary_all_nodes.txt 39051 bytes\n30_FINAL_AUDIT_GAPS_WARNINGS_SECRET_CLASSIFICATION.txt 4366 bytes\n31_HOMELAB_REFERENCE.md 1805 bytes\ndatacenter.cfg.before-canonical-migration-20260629T211046 63 bytes\ndatacenter.cfg.before-migration-network-20260629T210710 16 bytes\n\n\n\nРУССКОЕ_РЕЗЮМЕ_КЛАСТЕРА\n- Кластер: homelab, 3 узла, quorum OK.\n- LAN: pve01=[PRIVATE_IP], pve02=[PRIVATE_IP], pve03=[PRIVATE_IP].\n- Внутренняя сеть: pve01-int=[PRIVATE_IP], pve02-int=[PRIVATE_IP], pve03-int=[PRIVATE_IP].\n- Миграции Proxmox: migration: secure,network=[PRIVATE_IP]/24.\n- Corosync остаётся на [PRIVATE_IP]/12/13; не переносить без отдельного rollback-плана.\n\nРИСКИ_И_ДЕЙСТВИЯ\n- VM160 forum-prod не входит в ночной Proxmox backup.\n- У VM130 edge-vm есть риск: дополнительный диск backup=0.\n- Нужно ротировать ранее засвеченный Cloudflare token.\n- Желательно ротировать XenForo SMTP пароль, который был выведен до редактирования.\n- Проверить adguard-rewrite-sync: STATUS=ERROR, REASON=URLError.\n- Проверить backup-restore-dashboard: STATUS=WARN, NOT_OK=13.\n\nДОСТУПЫ_КРАТКО\n- Proxmox: ssh root@pve01, ssh root@pve02, ssh root@pve03.\n- Edge VM: ssh debian@[PRIVATE_IP], использовать sudo, root-login не использовать.\n- Nextcloud VM: ssh debian@[PRIVATE_IP].\n- Forum-prod: сначала ssh root@pve02, затем ssh -i [SENSITIVE_PATH] root@[PRIVATE_IP].\n\nНАГРУЗКИ_И_СЕРВИСЫ\n- CT110 dns1 pve01 [PRIVATE_IP] AdGuard Home.\n- CT111 dns2 pve02 [PRIVATE_IP] AdGuard Home.\n- CT112 unbound1 pve01 [PRIVATE_IP] Unbound.\n- CT113 unbound2 pve02 [PRIVATE_IP] Unbound.\n- VM130 edge-vm pve03 [PRIVATE_IP] Docker ingress/app host.\n- VM150 Nextcloud pve01 [PRIVATE_IP] Nextcloud AIO.\n- VM160 forum-prod pve02 [PRIVATE_IP] CodeVipe/XenForo.\n\nBACKUP_КРАТКО\n- Ночной Proxmox backup включает VMID 110,111,112,113,130,150.\n- VM160 forum-prod исключён из ночного Proxmox backup и держится на app-level local/cloud backup.\n- Edge VM имеет vzdump/offhost/restore evidence; риск backup=0 по дополнительному диску остаётся.\n- Nextcloud VM150 имеет Proxmox backup и restore-proof evidence.\n- CodeVipe forum-prod имеет локальный backup каждые 6 часов и cloud backup с pve02 каждые 6 часов.\n\nБЕЗОПАСНОСТЬ_КРАТКО\n- Финальная строгая проверка audit-файлов: STRICT_POSSIBLE_SECRET_COUNT=0.\n- Redacted DNS configs сохранены в 28_dns_configs_redacted.txt.\n- Справочник 31 проверен: REFERENCE_ASSIGNMENT_SECRET_HITS=0.\n\n## FINAL_CLOSURE_20260630_CRITICAL_TAILS\n\n- VM160 backup: closed, nightly includes VM160, manual backup ZSTD_OK.\n- VM130 backup: closed, scsi1 backup=1, manual backup ZSTD_OK, exact offhost ZSTD_OK, old local backup removed.\n- AdGuard rewrite sync: closed, STATUS=OK, timer active/enabled.\n- Cloudflare token: live NPMplus token rotated, API verify OK, old exposed token externally confirmed revoked.\n- XenForo SMTP: password rotated, JSON valid, STARTTLS SMTP AUTH OK.\n- Final audit: unredacted secret strict scan OK.\n- Proof index: 85_FINAL_CLOSURE_PROOF.txt and 87_FINAL_CLOSURE_PROOF_WITH_EXTERNAL_CONFIRMATION.txt.\n\n## FINAL_DASHBOARD_RUNTIME_OK_20260630\n\n- Runtime dashboard: STATUS=OK, ITEMS=69, NOT_OK=0.\n- systemd failed units: 0 loaded units listed.\n- SLO/coverage WARN files: moved out of runtime dashboard into slo-coverage-backlog-index.txt.\n- Backlog index: STATUS=OK, BACKLOG_WARN_COUNT=12, RUNTIME_BLOCKER=NO.\n- Final proof: 103_FINAL_ALL_GREEN_DASHBOARD_PROOF.txt.\n## ROUTER_NETCRAZE_ULTRA_NC1812_20260630\n\nИсточник: self-test Netcraze Ultra NC-1812 от 2026-06-30. Секреты Wi-Fi, пароли и токены в справочник не вносятся.\n\n### Паспорт\n- Model: Netcraze Ultra.\n- Device description: Netcraze Ultra (NC-1812).\n- Hostname: Netcraze-9202.\n- Workgroup/domain: WORKGROUP.\n- Timezone: Europe/Moscow.\n- NTP: master.\n- NDNS/caption: ndns-domain.\n- Локальный домен/сертификат в логах: aleisaevn.netcraze.pro.\n- Firmware/NDM из self-test: 5.01.C.0.0-1, built 2026-06-17.\n- sharing-config version: 2.06.1.\n- Components auto-update: disabled.\n- Auto-update channel: draft.\n- Auto-update schedule0: start 05:00, stop 06:00.\n- EasyConfig: disabled.\n- zram: enabled.\n- IPv4 forwarding: enabled.\n- IPv6 forwarding: enabled.\n- conntrack max entries: 32768.\n- TCP established timeout: 1200.\n- TCP fin timeout: 30.\n- TCP keepalive: 120.\n\n### WAN и резервный интернет\n- Main WAN: GigabitEthernet1, renamed ISP, description Ростел.\n- WAN security-level: public.\n- WAN addressing: DHCP.\n- WAN MTU: 1500.\n- WAN global priority: 700.\n- WAN ping-check profile: default.\n- Ping-check hosts from log/profile: ya.ru, vk.com, dzen.ru over TLS/443.\n- WAN DHCP observed in router log: 95.84.154.183/23, gateway 95.84.154.1.\n- ISP DNS observed in router log: 77.37.251.33, 77.37.255.30.\n- Backup/mobile WAN: CdcEthernet0, description SIM.\n- CdcEthernet0 USB device-id: 12d1 14dc.\n- CdcEthernet0 security-level: public.\n- CdcEthernet0 addressing: DHCP.\n- CdcEthernet0 global priority: 350.\n- CdcEthernet0 observed IP: [PRIVATE_IP]/24 via [PRIVATE_IP].\n\n### LAN / VLAN / bridge\n- Home bridge: Bridge0 renamed Home.\n- Home description: Основная.\n- Home security-level: private.\n- Home IP: [PRIVATE_IP]/24.\n- Home includes: GigabitEthernet0/Vlan1, AccessPoint, AccessPoint_5G, XGigabitEthernet0.\n- Proxmox bridge: Bridge1.\n- Proxmox bridge IP: [PRIVATE_IP]/24.\n- Proxmox security-level: protected.\n- Proxmox includes: GigabitEthernet0/Vlan50, XGigabitEthernet0/Vlan50.\n- Port 1: GigabitEthernet0/0, access VLAN 50.\n- Port 2: GigabitEthernet0/1, access VLAN 1 + trunk VLAN 50.\n- Port 3: GigabitEthernet0/2, access VLAN 1 + trunk VLAN 50.\n- Port 4: GigabitEthernet0/3, access VLAN 1 + trunk VLAN 50.\n- Port 5: XGigabitEthernet0/0, XGigabitEthernet0/Vlan50 enabled.\n\n### Wi-Fi\n- SSID: N9202.\n- 2.4 GHz: WifiMaster0, AccessPoint.\n- 2.4 GHz compatibility: BGN+AX+BE.\n- 2.4 GHz channel width: 40-below.\n- 5 GHz: WifiMaster1, AccessPoint_5G.\n- 5 GHz compatibility: AN+AC+AX+BE.\n- 5 GHz channel width: 160.\n- Auto channel rescan: 00:00 interval 1 hour.\n- Encryption: WPA2 + WPA3.\n- WMM: enabled.\n- Beamforming: enabled.\n- TWT: enabled.\n- DL/UL MU-MIMO: enabled.\n- DL/UL OFDMA: enabled.\n- Spatial reuse: enabled.\n- Band steering: disabled.\n- Extra AP interfaces: present but down.\n- MWS WLAN Home: bands 0 and 1, bind Home, SSID N9202, encryption wpa2+3.\n\n### DHCP\n- Main DHCP pool: _WEBADMIN.\n- DHCP range: [PRIVATE_IP]-[PRIVATE_IP].\n- Default router: [PRIVATE_IP].\n- DHCP DNS: [PRIVATE_IP], [PRIVATE_IP].\n- Lease: 25200 seconds.\n- Bound interface: Home.\n- Guest AP pool: _WEBADMIN_GUEST_AP enabled.\n\n### Static DHCP reservations\n- [PRIVATE_IP] -> 00:e0:70:62:98:fb comp01.\n- [PRIVATE_IP] -> 60:3d:61:d4:57:ea Yandex-Ambi-Lamp.\n- [PRIVATE_IP] -> cc:4b:73:58:9b:c0 Yandex.Station.\n- [PRIVATE_IP] -> 38:8a:06:5b:eb:40 Мой смарт.\n- [PRIVATE_IP] -> a0:9d:c1:84:7b:98 Приставка комната.\n- [PRIVATE_IP] -> ac:ba:c0:51:d3:ca Алиса наша комната.\n- [PRIVATE_IP] -> 54:f1:5f:f3:cd:14 Приставка кухня.\n- [PRIVATE_IP] -> ac:ba:c0:50:71:94 Алиса кухня.\n- [PRIVATE_IP] -> 24:4b:fe:d8:8d:2c Коля смарт ROG-Phone3.\n- [PRIVATE_IP] -> 00:2b:70:9a:a3:cf Коля смарт S21-FE.\n- [PRIVATE_IP] -> 00:08:22:34:26:fc Коля SPIDER-10 первый.\n- [PRIVATE_IP] -> 6c:c7:ec:93:be:dc Света смарт.\n- [PRIVATE_IP] -> 00:1e:8f:b0:d5:af Принтер Canon ч/б.\n- [PRIVATE_IP] -> d8:43:ae:68:a3:a1 Коля комп.\n- [PRIVATE_IP] -> 80:1f:12:69:24:e6 Сигнализация.\n- [PRIVATE_IP] -> 00:2b:70:9a:9e:99 Мой смарт S21-FE.\n- [PRIVATE_IP] -> 74:eb:80:2e:aa:4e Света планшет Galaxy Tab S4.\n- [PRIVATE_IP] -> 84:47:09:6a:cc:ca PVE01.\n- [PRIVATE_IP] -> 6c:1f:f7:28:72:4f PVE02.\n- [PRIVATE_IP] -> 08:26:ae:3b:73:61 PVE03.\n- [PRIVATE_IP] -> bc:24:11:30:d4:0f AdGuard 1.\n- [PRIVATE_IP] -> bc:24:11:5b:7e:d8 AdGuard 2.\n- [PRIVATE_IP] -> bc:24:11:f4:c5:d8 WireGuard.\n- [PRIVATE_IP] -> bc:24:11:e1:f3:3c NPMplus / edge-vm.\n- [PRIVATE_IP] -> bc:24:11:e1:9a:25 Nextcloud.\n\n### NAT / port forwarding\n- ISP tcp/80 -> bc:24:11:e1:f3:3c, NPMplus.\n- ISP tcp/443 -> bc:24:11:e1:f3:3c, NPMplus.\n- ISP tcpudp/3478 -> bc:24:11:e1:9a:25, Nextcloud Talk TURN.\n- Home tcp/51820 -> bc:24:11:f4:c5:d8, WireGuard vpn1.\n\n### ACL / firewall\n- isolate-private enabled.\n- _WEBADMIN_Bridge0 permits Home-to-Proxmox access for DNS1/DNS2, AdGuard UI, NPMplus HTTP/HTTPS/UI, Nextcloud AIO/Talk TURN, Proxmox SSH/8006 and ICMP from admin PC.\n- _WEBADMIN_GigabitEthernet1 permits admin PC [PRIVATE_IP] outbound TCP 80/443.\n- Hotspot policy: Home permit, Bridge1 permit; known hosts explicitly permitted.\n\n### Router management\n- HTTP port: 5080.\n- HTTPS port: 5083.\n- HTTP/HTTPS security-level: private.\n- SSH port: 2222.\n- SSH security-level: private.\n- Telnet port configured: 2323, security-level private; router log shows Telnet disabled later on 2026-06-26.\n- Lockout policy for HTTP/Telnet/SSH: 5 15 3.\n- Cloud control2 security-level: private.\n- Admin tags: cli, http, cifs, printers, opt.\n- SFTP denied for admin in log; SSH CLI works.\n\n### Router services\n- service dhcp enabled.\n- service dns-proxy enabled.\n- service igmp-proxy enabled.\n- service http enabled.\n- service cifs enabled.\n- service ssh enabled.\n- service ntp enabled.\n- DNS proxy rebind protection: auto.\n- mDNS reflector: disabled.\n- UPnP LAN: Home.\n- DLNA interface: Home.\n- CIFS share: NTFS-edgtobxf, volume 6F98787822C01361:, automount, permissive.\n- USB storage seen in log: Huawei modem/storage, sdb1 swap around 2G, sdb2 NTFS volume NTFS-edgtobxf.\n\n### Router automation warning\n- Netcraze SSH CLI is not a normal POSIX shell.\n- Logs contain failed commands: while, unset, follow.\n- Automation must use router CLI syntax, not bash syntax.\n## EXTERNAL_SERVICES_UPS_NETBIRD_MAIL_SCRIPTS_20260630\n\n### UPS / NUT\n- UPS hardware discovered on pve01 USB: Cyber Power System CP1500 AVR UPS, USB ID 0764:0501.\n- pve01 role: NUT server + monitor.\n- pve01 enabled NUT units: nut-driver-enumerator, nut-monitor, nut-server, nut-driver.target, nut.target.\n- pve02 role: NUT monitor/client.\n- pve03 role: NUT monitor/client.\n- edge-vm: no UPS/NUT/APCUPSD integration discovered.\n- UPS detailed proof: 107_UPS_DISCOVERY_AND_SETTINGS.txt and 109_UPS_NUT_DETAILED_STATUS_AND_CONFIGS.txt.\n- Secrets from /etc/nut configs must remain redacted in audit output.\n\n### NetBird\n- NetBird service is active on pve01, pve02, pve03 and edge-vm.\n- NetBird version observed: daemon 0.73.2, CLI 0.73.2.\n- pve01: FQDN pve01.netbird.selfhosted, IPv4 100.100.131.41/16.\n- pve02: FQDN pve02.netbird.selfhosted, IPv4 100.100.7.2/16.\n- pve03: FQDN pve03.netbird.selfhosted, IPv4 100.100.34.141/16.\n- edge-vm: FQDN edge-vm.netbird.selfhosted, IPv4 100.100.60.182/16.\n- Interface: wt0, type Kernel.\n- WireGuard port: 51820.\n- Management/Signal: Connected.\n- Relays: 2/2 available.\n- SSH Server through NetBird: Disabled.\n- Observed peers count on listed hosts: 6/9 Connected.\n- Inventory proof: 108_EXTERNAL_SERVICES_NETBIRD_MAIL_SCRIPTS_INVENTORY.txt.\n\n### Forum mail / SMTP\n- forum-prod host: forum-prod.gram1.ru, VM IP [PRIVATE_IP].\n- Local MTA services were not discovered by the first inventory command; forum mail is treated as XenForo external SMTP, not local Postfix/Dovecot.\n- XenForo SMTP rotation proof: 80_xenforo_smtp_rotation_closed.txt.\n- SMTP auth proof: 79_forum_xenforo_smtp_auth_openssl_no_crlf_no_secret.txt.\n- Safe current SMTP inventory: 110_XENFORO_MAIL_EXTERNAL_SMTP_SAFE_INVENTORY.txt.\n- SMTP password must never be printed; store only host, port, SSL mode, auth flag, username length, password length and AUTH OK proof.\n\n### Scripts / automations\n- Full script/unit inventory proof: 108_EXTERNAL_SERVICES_NETBIRD_MAIL_SCRIPTS_INVENTORY.txt.\n- edge-vm owns most application health, dashboard, ingress, backup, NetBox, NPMplus, certificate, Trivy and vulnerability jobs.\n- pve01 owns many backup/offhost/restore/health/security/NetBird VPS/rclone/sops/scrutiny jobs.\n- pve02 owns CodeVipe cloud backup, Vaultwarden isolated restore drill, smartctl textfile and staging/rclone helpers.\n- pve03 owns smartctl textfile, cluster internal IP, NetBird and staging/rclone helpers.\n- All scripts must be treated as managed operational surface; every future edit needs proof file in the active cluster-audit directory.\n## UPS_NUT_DETAILED_CONFIG_20260630\n\n- USB discovery on pve01: Cyber Power System, Inc. CP1500 AVR UPS, USB ID 0764:0501.\n- NUT logical UPS name: cyberpower.\n- NUT-reported device model: BR1000ELCD.\n- NUT manufacturer: CPS.\n- Driver: usbhid-ups.\n- NUT driver version: 2.8.1.\n- CyberPower HID data version: 0.8.\n- NUT USB vendorid/productid: 0764/0501.\n- NUT server node: pve01.\n- NUT server mode: MODE=netserver.\n- NUT server listeners: 127.0.0.1:3493 and [PRIVATE_IP]:3493.\n- NUT clients: pve02 and pve03 in MODE=netclient.\n- pve01 monitor role: MONITOR cyberpower@localhost master.\n- pve02 monitor role: MONITOR cyberpower@[PRIVATE_IP] slave.\n- pve03 monitor role: MONITOR cyberpower@[PRIVATE_IP] slave.\n- Current UPS status at inventory time: OL.\n- Battery charge: 100%.\n- Battery warning threshold: 20%.\n- Battery low threshold: 10%.\n- Runtime estimate: 2544 seconds.\n- Runtime low threshold: 300 seconds.\n- Battery type: PbAcid.\n- Battery voltage: 12.9V nominal 12V.\n- Input voltage: 221.0V nominal 230V.\n- Output voltage: 221.0V.\n- UPS load: 11%.\n- Nominal real power: 600W.\n- Beeper: disabled.\n- Shutdown delay: 20 seconds.\n- Start delay: 30 seconds.\n- Shutdown command on monitored nodes: /sbin/shutdown -h +0.\n- NUT timing: POLLFREQ=5, POLLFREQALERT=5, HOSTSYNC=15, DEADTIME=30, FINALDELAY=5.\n- Powerdown flag: /etc/killpower.\n- Secrets in /etc/nut/upsd.users and upsmon.conf are intentionally redacted in proofs.\n- Proof file: 109_UPS_NUT_DETAILED_STATUS_AND_CONFIGS.txt.\n\n## XENFORO_EXTERNAL_SMTP_CURRENT_20260630\n\n- Host: forum-prod.gram1.ru, VM IP [PRIVATE_IP].\n- Local MTA units: none discovered in inventory output.\n- Mail mode: XenForo external SMTP, not local Postfix/Dovecot.\n- SMTP_HOST=mail.pvepro.ru\n- SMTP_PORT=587\n- SMTP_SSL=false\n- SMTP_AUTH=login\n- USERNAME_LEN=17\n- PASSWORD_LEN=30\n- SECRET_PRINTED=NO\n- SMTP AUTH proof: 79_forum_xenforo_smtp_auth_openssl_no_crlf_no_secret.txt.\n- Rotation closure proof: 80_xenforo_smtp_rotation_closed.txt.\n## TIMERS_AND_SCRIPT_HASHES_REFERENCE_20260630\n\n- Full timers and script SHA256 inventory: 114_TIMERS_AND_SCRIPT_HASHES_INVENTORY.txt.\n- edge-vm timers cover runtime dashboard, Paperless guard, external canary, health metrics, restore drill index, AdGuard rewrite sync, NPMplus cert expiry, NetBox backup/sync, retention, cluster daily status, vulnerability and Trivy scans, ingress hardening and NPMplus admin bind.\n- pve01 timers cover VPN/NetBird health, health metrics, smartctl, disk space, MkDocs refresh, VPS identity audit, storage capacity, quality gate, evidence catalog, backup freshness, docker health, Filebrowser backup/offhost/restore, NPMplus/Kuma backup, NetBird VPS backup/offhost, Authentik/Gitea/Vaultwarden backup, SOPS secret coverage, mail cloud upload/restore, Immich/Memos/Paperless backup/offhost/restore, auto backup, edge-vm vzdump, secret sanity.\n- pve02 timers cover smartctl, CodeVipe cloud backup, dpkg-db backup and Vaultwarden isolated restore drill.\n- pve03 timers cover smartctl and dpkg-db backup.\n- Script hashes were captured for /usr/local/sbin and /usr/local/bin on edge-vm, pve01, pve02 and pve03.\n- Future script edits must create a new proof file and refresh 114_TIMERS_AND_SCRIPT_HASHES_INVENTORY.txt.\n## PROXMOX_CLUSTER_NODE_STORAGE_REFERENCE_20260630\n\n### Cluster\n- Cluster name: homelab.\n- Nodes: 3.\n- Quorum: OK / Quorate Yes.\n- Expected votes: 3.\n- Quorum threshold: 2.\n- Transport: knet.\n- Secure auth: on.\n- Corosync membership: [PRIVATE_IP] pve01, [PRIVATE_IP] pve02, [PRIVATE_IP] pve03.\n- Proxmox VE: pve-manager 9.2.3 on all three nodes.\n- Debian: 13 / trixie on all three nodes.\n- Kernel: 7.0.12-1-pve on all three nodes.\n- Datacenter migration config: secure, network=[PRIVATE_IP]/24.\n- Internal migration/cluster helper IPs: pve01 [PRIVATE_IP], pve02 [PRIVATE_IP], pve03 [PRIVATE_IP].\n\n### Storage policy\n- Storage local: dir /var/lib/vz, content iso/vztmpl/backup/import.\n- Storage local-lvm: lvmthin pool data, content rootdir/images.\n- pve01 local usage at inventory: 32.56%, local-lvm 17.50%.\n- pve02 local usage at inventory: 12.86%, local-lvm 11.65%.\n- pve03 local usage at inventory: 35.79%, local-lvm 64.84%.\n- pve01 staging LV: /mnt/staging, 300G.\n- pve02 staging LV: /mnt/staging, 150G.\n- pve03 staging LV: /mnt/staging, 200G.\n- pve03 local-lvm is the most constrained active VM storage because VM130 has 96G OS disk plus 150G media disk.\n\n### Nightly Proxmox backup job\n- Job: homelab-nightly-all.\n- Schedule: 03:30.\n- Mode: snapshot.\n- Compression: zstd.\n- Storage: local.\n- Enabled: yes.\n- Mail notification: failure.\n- Included VMIDs: 110,111,112,113,130,150,160.\n- Retention: keep-daily=7, keep-weekly=4, keep-monthly=3.\n\n### Node pve01\n- FQDN: pve01.gram1.ru.\n- LAN IP: [PRIVATE_IP]/24.\n- Internal migration IP: [PRIVATE_IP]/24.\n- NetBird IP: 100.100.131.41/16.\n- Bridge: vmbr0 over nic0.\n- CPU: Intel Core i9-12950HX, 24 logical CPUs.\n- RAM: 31Gi.\n- Disk: Lexar SSD NQ7A1 1TB.\n- Main active workloads: CT110 dns1, CT112 unbound1, VM150 nextcloud.\n\n### Node pve02\n- FQDN: pve02.gram1.ru.\n- LAN IP: [PRIVATE_IP]/24.\n- Internal migration IP: [PRIVATE_IP]/24.\n- NetBird IP: 100.100.7.2/16.\n- Bridge: vmbr0 over nic2.\n- CPU: Intel N100, 4 logical CPUs.\n- RAM: 15Gi.\n- Disk: SK800-1TB.\n- Main active workloads: CT111 dns2, CT113 unbound2, VM160 forum-prod.\n\n### Node pve03\n- FQDN: pve03.gram1.ru.\n- LAN IP: [PRIVATE_IP]/24.\n- Internal migration IP: [PRIVATE_IP]/24.\n- NetBird IP: 100.100.34.141/16.\n- Bridge: vmbr0 over nic1.\n- CPU: Intel Core i7-4900MQ, 8 logical CPUs.\n- RAM: 31Gi.\n- Disk: Samsung SSD 870 EVO 500GB.\n- Main active workload: VM130 edge-vm.\n\n### Proof\n- Full raw inventory: 120_PROXMOX_CLUSTER_NODE_STORAGE_NETWORK_INVENTORY.txt.\n## PROXMOX_VM_CT_INVENTORY_REFERENCE_20260630\n\n### CT110 dns1\n- Type: LXC.\n- Node: pve01.\n- Hostname/name: dns1.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:F9:3C:E1.\n- Role: AdGuard Home DNS primary.\n- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameserver: [PRIVATE_IP].\n- Onboot: yes.\n- Startup order: 30.\n- Unprivileged: yes.\n- Backup job: included in homelab-nightly-all.\n\n### CT111 dns2\n- Type: LXC.\n- Node: pve02.\n- Hostname/name: dns2.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:CF:3F:66.\n- Role: AdGuard Home DNS secondary.\n- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameserver: [PRIVATE_IP].\n- Onboot: yes.\n- Startup order: 30.\n- Unprivileged: yes.\n- Backup job: included in homelab-nightly-all.\n\n### CT112 unbound1\n- Type: LXC.\n- Node: pve01.\n- Hostname/name: unbound1.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:22:A6:0D.\n- Role: Unbound recursive resolver primary.\n- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameserver: [PRIVATE_IP].\n- Onboot: yes.\n- Startup order: 20.\n- Unprivileged: yes.\n- Backup job: included in homelab-nightly-all.\n\n### CT113 unbound2\n- Type: LXC.\n- Node: pve02.\n- Hostname/name: unbound2.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:9E:AF:BE.\n- Role: Unbound recursive resolver secondary.\n- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameserver: [PRIVATE_IP].\n- Onboot: yes.\n- Startup order: 20.\n- Unprivileged: yes.\n- Backup job: included in homelab-nightly-all.\n\n### VM130 edge-vm\n- Type: QEMU VM.\n- Node: pve03.\n- Name: edge-vm.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:E1:F3:3C.\n- User: debian.\n- Role: edge application host / reverse proxy / monitoring / backup automation host.\n- Resources: 4 cores, 12GiB RAM.\n- Disks: scsi0 96G OS, scsi1 150G media/data.\n- scsi1 backup flag: backup=1.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameservers: [PRIVATE_IP], [PRIVATE_IP].\n- QEMU guest agent: enabled.\n- Onboot: yes.\n- Startup order: 40.\n- Backup job: included in homelab-nightly-all.\n- Important note: VM130 has exact offhost backup proof after scsi1 backup=1.\n\n### VM150 nextcloud\n- Type: QEMU VM.\n- Node: pve01.\n- Name: nextcloud.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:E1:9A:25.\n- User: debian.\n- Role: Nextcloud AIO.\n- Resources: 4 cores, 8GiB RAM.\n- Disk: scsi0 64G.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameservers: [PRIVATE_IP], [PRIVATE_IP].\n- QEMU guest agent: enabled.\n- Onboot: yes.\n- Startup order: 50.\n- Backup job: included in homelab-nightly-all.\n\n### VM160 forum-prod\n- Type: QEMU VM.\n- Node: pve02.\n- Name: forum-prod.\n- IP: [PRIVATE_IP]/24.\n- MAC: BC:24:11:B6:45:ED.\n- User: ops.\n- Role: CodeVipe / XenForo production forum.\n- Resources: 2 cores, 4GiB RAM.\n- Disk: scsi0 80G.\n- Network: vmbr0, gateway [PRIVATE_IP].\n- Nameserver in VM config: 1.1.1.1.\n- QEMU guest agent: enabled.\n- Onboot: yes.\n- Startup order: 30.\n- Backup job: included in homelab-nightly-all.\n- Important note: VM160 manual backup proof exists and VM160 is included in nightly job.\n\n### Proof\n- Redacted config inventory: 121_PROXMOX_VM_CT_CONFIG_INVENTORY_REDACTED.txt.\n## DNS_INGRESS_CERTIFICATES_REFERENCE_20260630\n\n### DNS chain\n- Home DHCP gives clients DNS servers: [PRIVATE_IP] and [PRIVATE_IP].\n- dns1: CT110, AdGuard Home, IP [PRIVATE_IP].\n- dns2: CT111, AdGuard Home, IP [PRIVATE_IP].\n- unbound1: CT112, IP [PRIVATE_IP], listens on [PRIVATE_IP]:5335 and 127.0.0.1:5335.\n- unbound2: CT113, IP [PRIVATE_IP], listens on [PRIVATE_IP]:5335 and 127.0.0.1:5335.\n- dns1 upstream: [PRIVATE_IP]:5335.\n- dns2 upstream: [PRIVATE_IP]:5335.\n- AdGuard bootstrap DNS: 1.1.1.1 and 9.9.9.9.\n- AdGuard filtering_enabled=true, rewrites_enabled=true, protection_enabled=true.\n- AdGuard ratelimit=20.\n- Unbound access model: localhost allowed, matching AdGuard IP allowed, rest of [PRIVATE_IP]/24 refused.\n- Unbound do-ip6: no.\n- Unbound private-address includes [PRIVATE_IP]/16, [PRIVATE_IP]/12 and [PRIVATE_IP]/8.\n\n### AdGuard rewrites\n- turn.gram1.ru -> [PRIVATE_IP].\n- git.gram1.ru -> [PRIVATE_IP].\n- dozzle.gram1.ru -> [PRIVATE_IP].\n- paper.gram1.ru -> [PRIVATE_IP].\n- memos.gram1.ru -> [PRIVATE_IP].\n- photos.gram1.ru -> [PRIVATE_IP].\n- auth.gram1.ru -> [PRIVATE_IP].\n- backup.gram1.ru -> [PRIVATE_IP].\n- Rewrite sync health proof: adguard-rewrite-sync STATUS=OK, ORIGIN_COUNT=8, REPLICA_COUNT=8, CHANGED=0.\n- Proof files: 127_ADGUARD_UNBOUND_DNS_SAFE_INVENTORY.txt and 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt.\n\n### Router ingress\n- Public WAN router forwards TCP/80 and TCP/443 to NPMplus on edge-vm, [PRIVATE_IP].\n- Router forwards TCP/UDP 3478 to Nextcloud Talk TURN on [PRIVATE_IP].\n- Router forwards Home TCP/51820 to WireGuard host [PRIVATE_IP].\n- NPMplus admin listener is bound to localhost on edge-vm, 127.0.0.1:81; public disabled legacy host npm.gram1.ru exists but enabled=0.\n\n### NPMplus runtime\n- Host: edge-vm, [PRIVATE_IP].\n- Container: npmplus, image zoeyvid/npmplus:2026-06-17-b1, healthy at inventory time.\n- Socket proxy: lscr.io/linuxserver/socket-proxy:latest, 127.0.0.1:2375.\n- Database: /opt/npmplus/npmplus/database.sqlite.\n- DB integrity: ok.\n- Public listen ports on edge-vm: 0.0.0.0:80 and 0.0.0.0:443 by nginx/NPMplus.\n- NPMplus admin: 127.0.0.1:81.\n- Secret rule: Cloudflare DNS API token exists only inside NPMplus certificate metadata and must never be printed.\n\n### Public NPMplus proxy hosts\n- nc.gram1.ru -> http://[PRIVATE_IP]:11000, cert=17, ssl_forced=1, enabled=1.\n- uptime.gram1.ru -> http://127.0.0.1:3001, cert=35, ssl_forced=1, enabled=1.\n- gotify.gram1.ru -> http://127.0.0.1:8082, cert=35, ssl_forced=1, enabled=1.\n- vault.gram1.ru -> http://127.0.0.1:8083, cert=21, ssl_forced=1, enabled=1.\n- dockge.gram1.ru -> http://127.0.0.1:5001, cert=26, ssl_forced=1, enabled=1.\n- home.gram1.ru -> http://127.0.0.1:3000, cert=27, ssl_forced=1, enabled=1.\n- git.gram1.ru -> http://127.0.0.1:3002, cert=29, ssl_forced=1, enabled=1.\n- dozzle.gram1.ru -> http://127.0.0.1:9999, cert=30, ssl_forced=1, enabled=1.\n- paper.gram1.ru -> http://127.0.0.1:8010, cert=30, ssl_forced=1, enabled=1.\n- memos.gram1.ru -> http://127.0.0.1:5230, cert=30, ssl_forced=1, enabled=1.\n- photos.gram1.ru -> http://127.0.0.1:2283, cert=30, ssl_forced=1, enabled=1.\n- auth.gram1.ru -> http://127.0.0.1:9000, cert=30, ssl_forced=1, enabled=1.\n- backup.gram1.ru -> http://[PRIVATE_IP]:9101, cert=31, ssl_forced=1, enabled=1.\n\n### VPN NPMplus proxy hosts\n- kuma.vpn.gram1.ru -> http://127.0.0.1:3001, cert=32.\n- homepage.vpn.gram1.ru -> http://127.0.0.1:3000, cert=32.\n- gitea.vpn.gram1.ru -> http://127.0.0.1:3002, cert=32.\n- filebrowser.vpn.gram1.ru -> http://127.0.0.1:8085, cert=32.\n- auth.vpn.gram1.ru -> http://127.0.0.1:9000, cert=32.\n- actual-budget.vpn.gram1.ru -> http://127.0.0.1:5006, cert=32.\n- grafana.vpn.gram1.ru -> http://127.0.0.1:3003, cert=32.\n- homebox.vpn.gram1.ru -> http://127.0.0.1:7745, cert=32.\n- mealie.vpn.gram1.ru -> http://127.0.0.1:9001, cert=32.\n- n8n.vpn.gram1.ru -> http://127.0.0.1:5678, cert=32.\n- netbox.vpn.gram1.ru -> http://127.0.0.1:8000, cert=32.\n- node-red.vpn.gram1.ru -> http://127.0.0.1:1880, cert=32.\n- beszel.vpn.gram1.ru -> http://127.0.0.1:8090, cert=32.\n- it-tools.vpn.gram1.ru -> http://127.0.0.1:8084, cert=32.\n- karakeep.vpn.gram1.ru -> http://127.0.0.1:3030, cert=32.\n- linkding.vpn.gram1.ru -> http://127.0.0.1:9091, cert=32.\n- minio.vpn.gram1.ru -> http://127.0.0.1:9003, cert=32.\n- ntfy.vpn.gram1.ru -> http://127.0.0.1:8055, cert=32.\n- searxng.vpn.gram1.ru -> http://127.0.0.1:8888, cert=32.\n- syncthing.vpn.gram1.ru -> http://127.0.0.1:8384, cert=32.\n- alertmanager.vpn.gram1.ru -> http://127.0.0.1:9093, cert=32.\n- prometheus.vpn.gram1.ru -> http://127.0.0.1:9090, cert=32.\n- homeassistant.vpn.gram1.ru -> http://127.0.0.1:8123, cert=32.\n- npmplus.vpn.gram1.ru -> https://[PRIVATE_IP]:81, cert=32.\n- proxmox.vpn.gram1.ru -> https://[PRIVATE_IP]:8006, cert=32.\n- dns1.vpn.gram1.ru -> http://[PRIVATE_IP]:8080, cert=32.\n- dns2.vpn.gram1.ru -> http://[PRIVATE_IP]:8080, cert=32.\n- checks.vpn.gram1.ru -> http://127.0.0.1:8015, cert=32.\n- vikunja.vpn.gram1.ru -> http://127.0.0.1:8016, cert=32.\n- bookstack.vpn.gram1.ru -> http://127.0.0.1:8017, cert=32.\n- pdf.vpn.gram1.ru -> http://127.0.0.1:8018, cert=32.\n- jellyfin.vpn.gram1.ru -> http://127.0.0.1:8019, cert=32.\n- audiobooks.vpn.gram1.ru -> http://127.0.0.1:8020, cert=32.\n- books.vpn.gram1.ru -> http://127.0.0.1:8021, cert=32.\n\n### NPMplus certificates\n- cert=17: nc.gram1.ru, expires 2026-09-12 21:58:35.\n- cert=21: vault.gram1.ru, expires 2026-09-13 13:43:23.\n- cert=26: dockge.gram1.ru, expires 2026-09-13 14:51:59.\n- cert=27: home.gram1.ru, expires 2026-09-13 16:43:44.\n- cert=29: git.gram1.ru, expires 2026-09-13 17:36:55.\n- cert=30: dozzle.gram1.ru, paper.gram1.ru, memos.gram1.ru, photos.gram1.ru, auth.gram1.ru, expires 2026-09-13 23:12:00.\n- cert=31: backup.gram1.ru, expires 2026-09-14 05:04:53.\n- cert=32: *.vpn.gram1.ru and vpn.gram1.ru, expires 2026-09-15 23:18:29.\n- cert=35: uptime.gram1.ru and gotify.gram1.ru, expires 2026-09-19 21:21:59.\n- Deleted/old cert rows observed: cert=1 nc.gram1.ru, cert=18 vault.gram1.ru.\n\n### Proof\n- NPMplus schema/listen inventory: 126_DNS_INGRESS_CERT_NPMPLUS_SAFE_INVENTORY.txt.\n- NPMplus exact proxy/cert rows: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt.\n- AdGuard/Unbound inventory: 127_ADGUARD_UNBOUND_DNS_SAFE_INVENTORY.txt.\n- AdGuard upstream/rewrites and Unbound rules: 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt.\n## EDGE_DOCKER_STACKS_REFERENCE_20260630\n\n### Runtime\n- Host: edge-vm, IP [PRIVATE_IP].\n- Docker Server version: 29.6.0.\n- Docker Compose version: v5.1.4.\n- Primary stack root: /opt/stacks.\n- Additional compose roots: /opt/npmplus-compose, /opt/gotify-compose, /opt/uptime-kuma-compose, /opt/vaultwarden-compose, /opt/dockge-compose.\n- Public ingress terminates through NPMplus on ports 80/443.\n- Most app containers expose only 127.0.0.1 ports and are published through NPMplus.\n- NPMplus uses host networking.\n- socket-proxy exposes Docker API read-oriented surface on 127.0.0.1:2375 and socket-proxy_net.\n- Secret values are not stored in the reference. Only .env/secret file paths are inventoried.\n\n### Compose projects observed\n- actual-budget.\n- audiobookshelf.\n- authentik.\n- beszel.\n- blackbox-exporter.\n- bookstack.\n- cadvisor.\n- calibre-web.\n- crowdsec.\n- diun.\n- dockge-compose.\n- dozzle.\n- filebrowser.\n- gitea.\n- gotify-compose.\n- healthchecks.\n- homeassistant.\n- homebox.\n- homepage.\n- immich.\n- it-tools.\n- jellyfin.\n- karakeep.\n- kopia.\n- linkding.\n- loki-alloy.\n- mealie.\n- memos.\n- minio.\n- n8n.\n- netbox.\n- node-red.\n- npmplus.\n- ntfy.\n- observability-lite.\n- paperless.\n- searxng.\n- socket-proxy.\n- stirling-pdf.\n- syncthing.\n- uptime-kuma-compose.\n- vaultwarden-compose.\n- vikunja.\n\n### Critical app groups\n- Ingress/security: npmplus, socket-proxy, crowdsec.\n- Identity/secrets: authentik, vaultwarden.\n- Monitoring/alerting: uptime-kuma, gotify, healthchecks, observability-lite, loki-alloy, blackbox-exporter, cadvisor, node-exporter, dozzle, diun.\n- Data apps: immich, paperless, memos, gitea, netbox, actual-budget, homebox, mealie, n8n, node-red, vikunja, bookstack.\n- Files/media: nextcloud external VM, immich, filebrowser, syncthing, jellyfin, audiobookshelf, calibre-web, stirling-pdf, minio.\n- Admin/dashboard: homepage, dockge, netbox, dozzle, NPMplus VPN admin.\n\n### Notable local ports\n- Homepage: 127.0.0.1:3000.\n- Uptime Kuma: 127.0.0.1:3001.\n- Gitea: 127.0.0.1:3002.\n- Grafana: 127.0.0.1:3003.\n- Actual Budget: 127.0.0.1:5006.\n- n8n: 127.0.0.1:5678.\n- Paperless: 127.0.0.1:8010.\n- Healthchecks: 127.0.0.1:8015.\n- Vikunja: 127.0.0.1:8016.\n- BookStack: 127.0.0.1:8017.\n- Stirling PDF: 127.0.0.1:8018.\n- Jellyfin: 127.0.0.1:8019.\n- Audiobookshelf: 127.0.0.1:8020.\n- Calibre-Web: 127.0.0.1:8021.\n- ntfy: 127.0.0.1:8055.\n- Gotify: 127.0.0.1:8082.\n- Vaultwarden: 127.0.0.1:8083.\n- IT-Tools: 127.0.0.1:8084.\n- Filebrowser: 127.0.0.1:8085.\n- Beszel: 127.0.0.1:8090.\n- Prometheus: 127.0.0.1:9090.\n- Linkding: 127.0.0.1:9091.\n- Alertmanager: 127.0.0.1:9093.\n- Node exporter: 127.0.0.1:9100.\n- Blackbox exporter: 127.0.0.1:9115.\n- Syncthing UI: 127.0.0.1:8384.\n- Loki: 127.0.0.1:3100.\n- Alloy UI: 127.0.0.1:12345.\n- Home Assistant: host network, 0.0.0.0:8123.\n- NPMplus admin: 127.0.0.1:81.\n- NPMplus public ingress: 0.0.0.0:80 and 0.0.0.0:443.\n\n### Secret/env inventory policy\n- Full values from .env, secrets.yaml, secretkey.txt and token files must never be copied into reference.\n- Reference may list paths only.\n- Full path inventory is in 133_EDGE_DOCKER_STACKS_SAFE_INVENTORY.txt.\n- Normalized runtime map is in 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt.\n\n### Proof\n- Raw Docker/stacks inventory: 133_EDGE_DOCKER_STACKS_SAFE_INVENTORY.txt.\n- Normalized container map: 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt.\n## BACKUP_RESTORE_OFFHOST_CATALOG_REFERENCE_20260630\n\n### Global backup model\n- Primary Proxmox backup job: homelab-nightly-all.\n- Schedule: daily 03:30.\n- Mode: snapshot.\n- Compression: zstd.\n- Storage: local.\n- Retention: keep-daily=7, keep-weekly=4, keep-monthly=3.\n- Included VMIDs: 110,111,112,113,130,150,160.\n- Backup health dashboard: STATUS=OK, ITEMS=69, NOT_OK=0 at 2026-06-30T10:10:06Z.\n- Restore drill index: STATUS=OK, RESTORE_PROOFS=16/16.\n- SLO coverage is tracked separately from runtime dashboard. It had WARN backlog in backup-restore-coverage-slo with APP_TOTAL=43, GREEN=3, YELLOW=21, RED=19, MISSING_BACKUP_SIGNAL=31, MISSING_RESTORE_SIGNAL=28, MISSING_TIMER=31.\n- Runtime OK means operational backup/restore checks are green; it does not mean every P1/P2 application has full backup+restore automation coverage.\n\n### Proxmox vzdump catalog\n- CT110 dns1: included in homelab-nightly-all, local backup on pve01.\n- CT111 dns2: included in homelab-nightly-all, local backup on pve02.\n- CT112 unbound1: included in homelab-nightly-all, local backup on pve01.\n- CT113 unbound2: included in homelab-nightly-all, local backup on pve02.\n- VM130 edge-vm: included in homelab-nightly-all, local backup on pve03.\n- VM150 nextcloud: included in homelab-nightly-all, local backup on pve01.\n- VM160 forum-prod: included in homelab-nightly-all, local backup on pve02.\n- VM130 also has dedicated edge-vm-vzdump backup/offhost/restore health proofs.\n- VM160 has manual backup proof and nightly job inclusion proof.\n- VM130 scsi1 backup flag is enabled after correction: scsi1 backup=1.\n\n### Edge VM / VM130 full-image protection\n- edge-vm-vzdump-backup: STATUS=OK, archive size about 28.2G, SHA256 recorded.\n- edge-vm-vzdump-offhost: STATUS=OK, destination pve02 /mnt/staging/offhost/edge-vm-vzdump-from-pve03.\n- edge-vm-vzdump-restore: STATUS=OK, zstd and vma verification OK.\n- mail-cloud-edge-vm: STATUS=OK, recurring chunked upload, 53 parts, download verification enabled.\n- Retention for edge-vm cloud upload: RETENTION_KEEP=4.\n- Edge restore proof offhost: STATUS=OK, ITEMS=14, OK=14.\n\n### Mail/cloud critical backups\n- mail-cloud-critical: STATUS=OK, recurring P0 critical baseline, remote OK 2/2.\n- Latest observed critical bundle: p0-critical-baseline-20260630T034644Z.tar.gz.\n- Critical bundle size at inventory: 1087208837 bytes.\n- Critical retention: RETENTION_KEEP=14.\n- mail-cloud-restore-drill: STATUS=OK, CRITICAL_OK=2, EDGE_OK=1.\n- vaultwarden isolated restore drill on pve02: STATUS=OK, uses cloud critical baseline plus pve02 offhost component and vaultwarden archive.\n\n### NPMplus / Kuma / ingress config backups\n- npmplus-kuma-config-backup: STATUS=OK.\n- Archive: /mnt/staging/npmplus-kuma-config-backups/snapshots/npmplus-kuma-config-*.tar.gz.\n- NPMplus DB integrity: OK.\n- Kuma DB integrity: OK.\n- NPM proxy count in health proof: 47.\n- Required VPN routes in health proof: 18.\n- Wildcard VPN cert check: NPM_CERT32_WILDCARD=1.\n- npmplus-kuma-config-offhost: STATUS=OK to pve02.\n- npmplus-kuma-config-restore: STATUS=OK with DB integrity checks.\n- npmplus restore proof also exists from app backup quality checks.\n\n### DNS / AdGuard / Unbound protection\n- AdGuard rewrite sync: STATUS=OK, ORIGIN_COUNT=8, REPLICA_COUNT=8, CHANGED=0.\n- dns1/dns2/unbound1/unbound2 are protected by Proxmox vzdump.\n- AdGuard rewrite sync plan stored under /var/lib/homelab-private/adguard-rewrite-sync.\n- Router self-test config is stored as uploaded/generated reference evidence, not yet automated as recurring router backup.\n\n### Auth / secrets / identity apps\n- Vaultwarden backup: STATUS=OK, archive under /mnt/staging/vaultwarden-backups/snapshots.\n- Vaultwarden offhost: STATUS=OK to pve02.\n- Vaultwarden restore: STATUS=OK, DB integrity OK, tables counted.\n- Vaultwarden isolated restore drill: STATUS=OK on pve02.\n- Authentik backup: STATUS=OK, archive under /mnt/staging/authentik-backups/snapshots.\n- Authentik offhost: STATUS=OK to pve02.\n- Authentik restore: STATUS=OK, tables/users/applications/proxy providers checked.\n- Secret values must remain out of all reference files; only archive paths, sizes and SHA256 are recorded.\n\n### Git / documentation / inventory apps\n- Gitea backup: STATUS=OK, archive under /mnt/staging/gitea-backups/snapshots.\n- Gitea offhost: STATUS=OK to pve02.\n- Gitea restore: STATUS=OK, DB integrity OK, tables counted.\n- NetBox backup: STATUS=OK, PostgreSQL dump under /var/lib/homelab-private/pre-update-backups/netbox-*.\n- NetBox offhost: STATUS=OK to pve02.\n- NetBox restore dry-run: STATUS=OK, restore container checked.\n- Latest NetBox health proof observed: devices=7, sites=1, prefixes=3, IP addresses=12, virtual machines=49.\n\n### Document / notes / personal data apps\n- Paperless backup: STATUS=OK, latest snapshot under /mnt/staging/paperless-backups/snapshots.\n- Paperless offhost: STATUS=OK to pve02.\n- Paperless restore dry-run: STATUS=OK, pg_restore list and restore tables checked.\n- Memos backup: STATUS=OK, latest snapshot under /mnt/staging/memos-backups/snapshots.\n- Memos offhost: STATUS=OK to pve02.\n- Memos restore proof exists from app restore quality checks.\n- Actual Budget backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof.\n- Linkding backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof.\n- Karakeep backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof.\n- Mealie backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof.\n- n8n backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof.\n\n### Files / media / sync apps\n- Immich media backup: STATUS=OK, latest snapshot under /mnt/staging/immich-media-backups/snapshots.\n- Immich media offhost: STATUS=OK to pve02.\n- Immich media restore: STATUS=OK, local/offhost manifest match.\n- Immich full consistency backup: STATUS=OK, includes PostgreSQL dump and media archive, offhost path present, restore dry-run OK.\n- Nextcloud restore proof: STATUS=OK, OCC OK, DB_DUMP OK, DATA_MANIFEST OK, restore to production NO.\n- Nextcloud restore proof is copied offhost.\n- Filebrowser backup: STATUS=OK.\n- Filebrowser offhost: STATUS=OK.\n- Filebrowser restore validation: STATUS=OK.\n- Jellyfin restore: STATUS=OK from app backup quality checks.\n- Audiobookshelf restore: STATUS=OK from app backup quality checks.\n- Calibre-Web restore: STATUS=OK from app backup quality checks.\n- Syncthing is inventoried in Docker/stacks; dedicated backup signal not confirmed in this catalog.\n\n### Home/admin/utility apps\n- Home Assistant backup: STATUS=OK, archive includes configuration.yaml, storage dir and DB file, offhost SHA256/listing OK, restore not attempted in observed proof.\n- Healthchecks restore: STATUS=OK, SQLite integrity OK and temp restore verify OK.\n- BookStack restore: STATUS=OK, DB dump OK.\n- Vikunja restore: STATUS=OK, SQLite integrity OK and temp restore verify OK.\n- Kopia restore: STATUS=OK, synthetic snapshot and restore compare OK.\n- Observability config backup: STATUS=OK, includes prometheus.yml, alertmanager.yml and health rules, offhost SHA256/listing OK.\n- Ntfy is covered as part of observability/alerting stack but dedicated service backup signal was not confirmed in this catalog.\n- MinIO is included in app-stacks-batch backup from 20260617; current dedicated timer was not confirmed.\n- Homebox, Node-RED, MinIO, Mealie, Linkding, n8n and Actual Budget were part of app-stacks batch backup proof from 20260617.\n\n### External service backups\n- NetBird VPS backup: STATUS=OK, snapshot under /mnt/staging/netbird-vps-backups/snapshots.\n- NetBird VPS offhost: STATUS=OK to pve02.\n- NetBird VPS restore validation: STATUS=OK, archive SHA256 recorded.\n- CodeVipe/forum-prod: VM160 Proxmox backup is present; pve02 also has codevipe-cloud-backup.timer.\n- XenForo SMTP rotation has DB backup proof from earlier rotation work; production forum application-level backup catalog should be expanded separately with CodeVipe-specific DB/files/cron/public cutover details.\n\n### Retention and cleanup\n- Retention dry-run: STATUS=OK, production changed NO, destructive actions NO.\n- Disk retention policy: STATUS=OK, root used pct observed 70 on edge-vm, removed dirs 0, Docker volume prune NO.\n- App backup retention dry-run timer exists on edge-vm.\n- homelab-backup-freshness timer exists on pve01.\n- rollback-image-retention health file exists but had permission issue during one raw read; it must be read with sudo in future collection commands.\n\n### Known coverage gaps / backlog\n- backup-restore-coverage-slo remains a coverage backlog, not a runtime blocker.\n- Actual Budget latest proof says RESTORE_ATTEMPTED=NO.\n- Home Assistant latest proof says RESTORE_ATTEMPTED=NO.\n- Linkding latest proof says RESTORE_ATTEMPTED=NO.\n- Karakeep latest proof says RESTORE_ATTEMPTED=NO.\n- Mealie latest proof says RESTORE_ATTEMPTED=NO.\n- n8n latest proof says RESTORE_ATTEMPTED=NO.\n- Observability config latest proof says RESTORE_ATTEMPTED=NO.\n- Router config backup is currently reference/self-test based, not confirmed as recurring automated backup.\n- Syncthing, ntfy, MinIO and several P2 services need either dedicated backup/restore proof or explicit classification as disposable/rebuildable.\n\n### Proof\n- Raw backup/offhost/restore inventory: 138_BACKUP_RESTORE_OFFHOST_CATALOG_RAW_INVENTORY.txt.\n- Backup timer summary draft: 139_BACKUP_CATALOG_SERVICE_SUMMARY_DRAFT.txt.\n- Normalized backup health index: 140_BACKUP_HEALTH_FILES_NORMALIZED_INDEX.txt.\n## MONITORING_ALERTING_HEALTH_REFERENCE_20260630\n\n### Monitoring stack\n- Primary monitoring host: edge-vm, [PRIVATE_IP].\n- Prometheus container: prometheus, local port 127.0.0.1:9090.\n- Grafana container: grafana, local port 127.0.0.1:3003, VPN route grafana.vpn.gram1.ru.\n- Alertmanager container: alertmanager, local port 127.0.0.1:9093.\n- Loki container: loki, local port 127.0.0.1:3100.\n- Alloy container: alloy, local port 127.0.0.1:12345.\n- Uptime Kuma container: uptime-kuma, local port 127.0.0.1:3001, public route uptime.gram1.ru and VPN route kuma.vpn.gram1.ru.\n- Gotify container: gotify, local port 127.0.0.1:8082, public route gotify.gram1.ru.\n- ntfy container: ntfy, local port 127.0.0.1:8055, VPN route ntfy.vpn.gram1.ru.\n- Healthchecks container: healthchecks, local port 127.0.0.1:8015, VPN route checks.vpn.gram1.ru.\n- Blackbox exporter: blackbox-exporter, local port 127.0.0.1:9115.\n- cAdvisor: cadvisor, local port 127.0.0.1:8081.\n- Node exporter on edge-vm: node-exporter, local port 127.0.0.1:9100.\n- Beszel: beszel, local port 127.0.0.1:8090.\n- Dozzle: dozzle, local port 127.0.0.1:9999.\n\n### PVE monitoring endpoints\n- pve01: prometheus-node-exporter active on :9100, homelab-health-http active on :9101, Proxmox pveproxy on :8006.\n- pve02: prometheus-node-exporter active on :9100, homelab-health-http active on :9101, Proxmox pveproxy on :8006.\n- pve03: prometheus-node-exporter active on :9100, Proxmox pveproxy on :8006.\n- pve03 has node-exporter/SMART timers but no homelab-health-http :9101 endpoint in observed output.\n- PVE smartctl textfile timers exist on pve01/pve02/pve03.\n- pve01 also runs private VPN host health, NetBird peer health, disk space health, MkDocs refresh, evidence catalog and quality gate timers.\n\n### Prometheus config\n- Prometheus scrape interval: 30s.\n- Prometheus evaluation interval: 30s.\n- Rule files path: /etc/prometheus/rules/*.yml.\n- Alertmanager target: alertmanager:9093.\n- Blackbox HTTP job targets:\n - https://nc.gram1.ru\n - https://git.gram1.ru\n - https://auth.gram1.ru\n - https://paper.gram1.ru\n - https://backup.gram1.ru\n- Edge node exporter scrape target: node-exporter:9100.\n- PVE node exporter targets: [PRIVATE_IP]:9100, [PRIVATE_IP]:9100, [PRIVATE_IP]:9100.\n- Proxmox exporter job targets pve01/pve02/pve03 through exporter endpoint [PRIVATE_IP]:9221.\n- cAdvisor scrape target: cadvisor:8080.\n\n### Alertmanager / notification routing\n- Alertmanager route receiver: ntfy.\n- Alertmanager webhook target: http://ntfy/homelab-alerts.\n- Alert group_by: alertname, instance, severity.\n- group_wait: 10s.\n- group_interval: 5m.\n- repeat_interval: 4h.\n- Alert routing proof: alert-routing.txt STATUS=OK, RULES_ADDED=6, PROMETHEUS_RELOAD=OK, TEST_NOTIFICATION_SENT=YES, NTFY_MESSAGES_PUBLISHED_DELTA=1.\n- alert-routing-health.txt is the standardized status alias and is STATUS=OK.\n\n### Core Prometheus alert rules\n- HomelabPrometheusTargetDown: up == 0 for 5m, severity warning.\n- HomelabSmartDiskHealthFailed: homelab_smartctl_health_passed == 0 for 10m, severity critical.\n- HomelabRootFilesystemLowSpace: root filesystem available below 15% for 15m, severity warning.\n- HomelabPrometheusConfigReloadFailed: prometheus_config_last_reload_successful == 0 for 5m, severity critical.\n- HomelabP0HealthStatusNotOk: homelab_health_status tier P0 equals 0 for 15m.\n- HomelabP0BackupRestoreHealthStale: P0 backup/offhost/restore health older than 36h.\n- HomelabP0WeeklyEdgeVmVzdumpHealthStale: weekly edge-vm vzdump health older than 8d.\n- HomelabP1BackupHealthStale: P1 backup health older than 7d.\n- HomelabMailCloudRestoreDrillStale: mail-cloud restore drill older than 8d.\n- HomelabMkdocsRuntimePublishStale: MkDocs runtime publish older than 30d.\n- HomelabHealthMetricsCollectorMissing: homelab health metrics absent from Prometheus.\n- HomelabExternalCanaryHealthNotOkOrStale: external canary not OK or older than 30m.\n- HomelabBlackboxProbeFailed / Missing: blackbox probe failed or absent.\n- HomelabVulnScanHealthNotOkOrStale: vulnerability scan failed or older than 8d.\n- HomelabNpmplusCertExpiryHealthNotOkOrStale: NPMplus cert expiry health failed or older than 48h.\n- HomelabBlackboxTlsCertificateExpiringSoon: TLS certificate expiry within 14 days.\n\n### Loki / Alloy log pipeline\n- Loki version observed: grafana/loki:3.7.2.\n- Alloy version observed: grafana/alloy:v1.17.0.\n- Loki auth_enabled: false.\n- Loki storage: local filesystem under /loki.\n- Loki schema: tsdb v13.\n- Loki retention_period: 14d.\n- Alloy discovers Docker through socket-proxy at tcp://socket-proxy:2375.\n- Alloy relabels container, compose_project, compose_service and host=edge-vm.\n- Alloy forwards Docker logs to http://loki:3100/loki/api/v1/push.\n- Loki readiness observed as \"ready\".\n\n### Runtime dashboard semantics\n- backup-restore-dashboard.txt is an authoritative runtime dashboard file.\n- Current observed runtime dashboard: STATUS=OK, ITEMS=69, NOT_OK=0.\n- backup-restore-dashboard.json confirmed not_ok=[].\n- restore-drill-index.txt observed: STATUS=OK, RESTORE_PROOFS=16/16, MISSING empty.\n- cluster-daily-status.txt observed: STATUS=OK, PROBLEM_COUNT=0.\n- external-canary.txt observed: STATUS=OK, BAD=0.\n- external canary checks include nc.gram1.ru, git.gram1.ru, auth.gram1.ru, backup.gram1.ru.\n- Alertmanager local API is reachable.\n- Gotify local /health returns green.\n- ntfy local /v1/health returns healthy true.\n- Uptime Kuma local endpoint returns HTTP 302 to /dashboard, which is expected for UI auth/redirect.\n- Healthchecks local status endpoint returns HTTP 301, which is treated as reachable.\n\n### Certificate / vulnerability health\n- npmplus-cert-expiry-health.txt is the standardized health alias for certificate expiry.\n- npmplus-certificate-expiry.txt is the detailed cert expiry file.\n- Detailed cert expiry file observed cert_count=9 and min_cert_days_left=75 with warn threshold 30.\n- cluster-daily-status local extra check reports npmplus-cert-expiry status OK, problems=0, min_days=75.\n- trivy-scan.txt observed: STATUS=OK, IMAGE_TOTAL=58, IMAGE_SCAN_OK=58, IMAGE_SCAN_ERROR=0, CRITICAL_TOTAL=112, HIGH_TOTAL=1650.\n- vuln-scan.txt observed: STATUS=OK, MODE=full, IMAGES=58, TRIVY_OK=49, CRITICAL_MATCHES=203.\n\n### SLO backlog semantics\n- Runtime dashboard OK does not mean SLO backlog is closed.\n- slo-coverage-backlog-index.txt observed: STATUS=OK, BACKLOG_WARN_COUNT=12, RUNTIME_BLOCKER=NO.\n- backup-restore-coverage-slo.txt observed: STATUS=WARN, APP_TOTAL=43, GREEN_COUNT=3, YELLOW_COUNT=21, RED_COUNT=19, MISSING_BACKUP_SIGNAL=31, MISSING_RESTORE_SIGNAL=28, MISSING_TIMER=31, MISSING_EXTERNAL_VALIDATION=43.\n- Other observed WARN backlog files include container-image-lifecycle-slo, cron-job-monitoring-slo, direct-heartbeat-pilot-readiness, healthchecks-heartbeat-coverage, healthchecks-job-coverage, healthchecks-job-monitoring-slo, homepage-service-catalog-slo, ingress-dns-route-slo, observability-health-surface-slo and security-vulnerability-slo.\n- SLO WARN files are tracked separately and are not runtime blockers unless explicitly promoted to P0 runtime checks.\n\n### Important file locations\n- Main health dir on edge-vm: /var/lib/homelab-health.\n- Prometheus config: /opt/stacks/observability-lite/prometheus/prometheus.yml.\n- Prometheus rules: /opt/stacks/observability-lite/prometheus/rules/.\n- Alertmanager config: /opt/stacks/observability-lite/alertmanager/alertmanager.yml.\n- Loki config: /opt/stacks/loki-alloy/loki-config.yaml.\n- Alloy config: /opt/stacks/loki-alloy/config.alloy.\n- Blackbox exporter config: /opt/stacks/blackbox-exporter/blackbox.yml.\n- Uptime Kuma DB: /opt/uptime-kuma-compose/data/kuma.db.\n- Gotify DB: /opt/gotify-compose/data/gotify.db.\n- ntfy DB/cache: /opt/stacks/ntfy/data/user.db and /opt/stacks/ntfy/cache/cache.db.\n- Healthchecks settings/data: /opt/stacks/healthchecks/local_settings.py and /var/lib/healthchecks.\n\n### Known caveats\n- A previous compact check looked for npmplus-cert-expiry.txt; use npmplus-certificate-expiry.txt for detailed cert expiry and npmplus-cert-expiry-health.txt for standardized health.\n- health-metrics.txt is not required to exist as a health file; health metrics may be emitted via node-exporter textfile collector.\n- Prometheus targets must be checked with explicit HTTP code and parsed JSON output, not a silent curl pipeline.\n\n### Proof\n- Raw monitoring inventory: 143_MONITORING_ALERTING_HEALTH_RAW_INVENTORY.txt.\n- Runtime/SLO/alerting status: 144_MONITORING_RUNTIME_SLO_ALERTING_STATUS.txt.\n- Kuma/Gotify/ntfy/Healthchecks status: 145_KUMA_GOTIFY_NTFY_HEALTHCHECKS_SAFE_STATUS.txt.\n- Corrected monitoring compact status: 146_MONITORING_COMPACT_STATUS_CORRECTED.txt.\n\n## PROMETHEUS_STATUS_CORRECTION_20260630\n\n- Initial monitoring compact check after reference creation reported Prometheus API HTTP_CODE=000 on 127.0.0.1:9090.\n- This was recorded as error-register item 30 and checked with proof files 149_PROMETHEUS_DIAG_AFTER_MONITORING_REFERENCE.txt, 150_PROMETHEUS_CONDITIONAL_RECOVERY.txt and 151_PROMETHEUS_REFERENCE_CORRECTION_PROOF.txt.\n- The Monitoring/Alerting section must be interpreted together with the latest Prometheus correction proof, not only with the earlier secret-scan proof.\n\n## COMMAND_PREFLIGHT_RULE_20260630\n\n- Strict operator rule: before every command, check both the error register and this reference file.\n- Required visible markers before main action: ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK.\n- If either check fails, do not run the main action.\n- Do not use exit 1 in interactive SSH sessions.\n- Proof: 152_COMMAND_PREFLIGHT_RULE_RECORDED.txt.\n\n## PROMETHEUS_TARGETS_SETTLED_20260630\n\n- Prometheus was absent after the initial monitoring reference proof and was restarted in 150_PROMETHEUS_CONDITIONAL_RECOVERY.txt.\n- Prometheus readiness proof exists in 151_PROMETHEUS_REFERENCE_CORRECTION_PROOF.txt.\n- Target health was rechecked with an additional settled scrape proof in 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt.\n- Interpret Prometheus status from the latest settled proof, not from the earlier immediate-after-start unknown target state.\n## PROMETHEUS_TARGETS_SETTLED_PROOF_CORRECTION_20260630\n\n- 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt is invalid and must not be used as evidence.\n- Reason: nested Python inside SSH lost quoting and produced SyntaxError.\n- Error-register item 32 records this mistake.\n- Valid replacement proof: 154_PROMETHEUS_TARGETS_SETTLED_PROOF_NO_PYTHON.txt.\n- Prometheus status must be interpreted from 154 and later proofs, not from 153.\n\n## MONITORING_PROMETHEUS_FINAL_CLOSED_20260630\n\n- Prometheus was initially absent after monitoring reference creation.\n- It was restarted and then verified after scrape settling.\n- Valid settled proof: 154_PROMETHEUS_TARGETS_SETTLED_PROOF_NO_PYTHON.txt.\n- Final observed state: HTTP_CODE=200, PROM_TARGETS_TOTAL=14, PROM_UP_COUNT=14, PROM_NON_UP_COUNT=0.\n- Correction/secret scan proof: 156_PROMETHEUS_CORRECTION_PROOF_AND_SECRET_SCAN.txt.\n- Invalid proof 153 must not be used.\n## SECURITY_ACCESS_SECRETS_OPERATING_MODEL_20260630\n\n### Command safety\n- Before every command, check /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md and current 31_HOMELAB_REFERENCE.md.\n- Print ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK before the main action.\n- Do not run the main action if either check fails.\n- Do not use exit 1 in interactive SSH sessions.\n- Avoid long nested SSH/Python/PHP/SQL quoting chains.\n- Do not print secrets.\n\n### Access model\n- Primary operator entrypoint: root@pve01 / [PRIVATE_IP].\n- pve02 and pve03 are reached as root from pve01.\n- edge-vm is reached as debian@[PRIVATE_IP] with sudo.\n- forum-prod is reached through pve02 using [SENSITIVE_PATH] to root@[PRIVATE_IP].\n- PVE users observed: root@pam and prometheus@pve.\n- prometheus@pve has PVEAuditor on / for Proxmox exporter access.\n\n### SSH and permissions\n- pve01 root keys observed: id_rsa, id_ed25519 and public keys.\n- pve02 root keys observed: id_rsa and forum-prod-ci-key.\n- pve03 root key observed: id_rsa.\n- edge-vm root key observed: id_ed25519; debian authorized_keys observed.\n- Security finding: pve01/pve02/pve03 root authorized_keys files were observed as mode 777.\n- Correction proof: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt.\n- Private key material must never be copied into the reference.\n\n### Secret storage\n- Primary private storage root: /var/lib/homelab-private.\n- Edge private secrets root: /var/lib/homelab-private/secrets.\n- NPMplus DB: /opt/npmplus/npmplus/database.sqlite, root-only mode observed.\n- Gotify DB: /opt/gotify-compose/data/gotify.db.\n- Uptime Kuma DB: /opt/uptime-kuma-compose/data/kuma.db.\n- Vaultwarden DB: /opt/vaultwarden-compose/data/db.sqlite3.\n- Rclone configs exist under root config paths and must not be printed.\n- Env/secret inventory is path-only: owner, mode, size and path only.\n\n### Network exposure\n- PVE nodes expose SSH :22, Proxmox :8006 and node-exporter :9100.\n- pve01/pve02 expose homelab-health-http :9101.\n- pve01 exposes NUT :3493 on localhost and [PRIVATE_IP].\n- edge-vm exposes public :80/:443 through NPMplus.\n- edge-vm NPMplus admin :81 and socket-proxy :2375 are bound to 127.0.0.1.\n- edge-vm registry cache :5000 is bound to [PRIVATE_IP].\n- edge-vm Home Assistant :8123 is intentionally LAN-exposed.\n- Edge ingress hardening proof reports STATUS=OK.\n\n### Rotation and scan proofs\n- Cloudflare token rotation completed without printing token values.\n- Old Cloudflare token revocation was externally confirmed.\n- XenForo SMTP password rotation completed without printing password values.\n- XenForo SMTP auth was verified with STARTTLS-safe method.\n- Current reference strict scan shows zero strict secret hits.\n- Selected generated reference blocks show zero strict secret hits.\n\n### Proof\n- Raw security/access inventory: 158_SECURITY_ACCESS_SECRETS_RAW_INVENTORY.txt.\n- Secret scan and rotation proof index: 159_SECURITY_SECRET_SCAN_AND_ROTATION_PROOF_INDEX.txt.\n- SSH permission correction proof: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt.\n## SECURITY_SSH_PERMISSION_CORRECTION_20260630\n\n- Initial proof 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt is not sufficient as closure evidence.\n- Reason: stat without -L still showed 777 after chmod, which may be symlink mode rather than target file mode.\n- Error-register item 35 records this proof-quality issue.\n- Authoritative replacement proof: 163_PVE_ROOT_AUTHORIZED_KEYS_SYMLINK_AWARE_FIX.txt.\n- Valid closure condition: target permissions checked with stat -L are 600 for authorized_keys files and [SENSITIVE_PATH] is 700.\n- Final observed status in replacement proof: STATUS=SSH_AUTHORIZED_KEYS_TARGET_PERMISSIONS_OK.\n## STORAGE_DISK_SMART_CAPACITY_REFERENCE_20260630\n\n### Proxmox storage model\n- Cluster storage is defined in /etc/pve/storage.cfg.\n- local storage: dir /var/lib/vz, content iso,vztmpl,backup,import.\n- local-lvm storage: lvmthin data, content rootdir,images.\n- Nightly vzdump job writes to local storage.\n- VM/CT images are primarily stored on local-lvm.\n- Staging/offhost paths are under /mnt/staging where present.\n\n### Node disks and capacity\n- pve01: primary disk previously inventoried as Lexar SSD NQ7A1 1TB.\n- pve02: primary disk previously inventoried as SK800-1TB.\n- pve03: primary disk previously inventoried as Samsung 870 EVO 500GB.\n- pve01 storage usage was previously observed around local 32.56% and local-lvm 17.50%.\n- pve02 storage usage was previously observed around local 12.86% and local-lvm 11.65%.\n- pve03 storage usage was previously observed around local 35.79% and local-lvm 64.84%.\n- Raw current df/lsblk/LVM/SMART evidence is in 166_STORAGE_DISK_SMART_CAPACITY_RAW_INVENTORY.txt.\n\n### Edge VM storage model\n- edge-vm runs Docker application stacks.\n- Docker app data lives mainly under /opt/stacks, /opt/npmplus, /opt/gotify-compose, /opt/uptime-kuma-compose, /opt/vaultwarden-compose and /var/lib application paths.\n- Immich media is mounted separately at /mnt/immich-media in the container mapping.\n- Docker volume and image usage is captured in docker system df output.\n- Disk retention health is tracked by disk-retention-policy and disk-retention-cleanup files.\n\n### SMART and health monitoring\n- PVE nodes run smartctl textfile timers.\n- pve01/pve02/pve03 expose node-exporter :9100 for Prometheus.\n- pve01 and pve02 expose homelab-health-http :9101.\n- SMART/disk/capacity health evidence is collected from /var/lib/homelab-health where present.\n- Prometheus alert rule HomelabSmartDiskHealthFailed watches homelab_smartctl_health_passed == 0.\n- Prometheus alert rule HomelabRootFilesystemLowSpace watches root filesystem available below 15%.\n- pve03 did not have /var/lib/homelab-health in previous backup-health collection, but it has node-exporter and smartctl timers.\n\n### Retention and cleanup\n- Edge disk retention policy previously observed STATUS=OK.\n- Edge disk retention policy observed ROOT_USED_PCT=70, REMOVED_DIRS=0, DOCKER_VOLUME_PRUNE=NO.\n- Retention dry-run policy is designed to avoid destructive production changes.\n- Broad Docker prune is not used as a default cleanup mechanism.\n- Cleanup and retention actions must have explicit proof files.\n\n### Operational rules\n- Before deleting or pruning storage, create or identify rollback/backup proof.\n- Do not delete vzdump/offhost/cloud backup artifacts unless retention policy and proof explicitly allow it.\n- Do not infer offhost success from a failed rsync.\n- For backup artifacts, verify exact file, size, checksum or zstd/vma integrity where applicable.\n- For SMART, use health output plus selected attributes, not full raw dumps with unnecessary noise.\n\n### Proof\n- Raw storage/disk/SMART/capacity inventory: 166_STORAGE_DISK_SMART_CAPACITY_RAW_INVENTORY.txt.\n- Earlier Proxmox node/storage inventory: 120_PROXMOX_CLUSTER_NODE_STORAGE_NETWORK_INVENTORY.txt.\n- Backup/restore catalog: 141_REFERENCE_BACKUP_RESTORE_OFFHOST_CATALOG_BLOCK.txt.\n- Monitoring rules and SMART alert model: 147_REFERENCE_MONITORING_ALERTING_HEALTH_BLOCK.txt.\n## STORAGE_CAPACITY_CORRECTION_20260630\n\n- Storage block integrity review proof: 169_STORAGE_BLOCK_INTEGRITY_AND_PVE03_CAPACITY_REVIEW.txt.\n- 167_REFERENCE_STORAGE_DISK_SMART_CAPACITY_BLOCK.txt integrity status: OK.\n- pve03 /mnt/staging current observed use percent: 77.\n- pve03 staging is close to the existing WARN threshold of 80% if observed use is 75% or higher.\n- pve01 disk-space.txt coverage visibly listed pve01_staging, pve02_staging and edge_immich_media; pve03_staging coverage must be added or explicitly documented elsewhere.\n- Storage layer is operationally OK only if SMART remains PASSED and pve03 staging is tracked before it crosses WARN/CRIT thresholds.\n- This is a capacity coverage note, not a secret or runtime outage.\n## PVE03_STAGING_CAPACITY_MONITOR_20260630\n\n- pve03 /mnt/staging was observed at 77% usage.\n- Existing pve01 disk-space.txt did not visibly include pve03_staging coverage.\n- A dedicated pve03 staging capacity health check was added on pve01.\n- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt.\n- Timer: homelab-pve03-staging-capacity-health.timer.\n- Service: homelab-pve03-staging-capacity-health.service.\n- Current observed status: OK.\n- WARN threshold: 80%.\n- CRIT threshold: 90%.\n- Proof: 173_PVE03_STAGING_CAPACITY_HEALTH_INSTALLED.txt.\n\n## STORAGE_LAYER_FINAL_CLOSED_WITH_PVE03_MONITOR_20260630\n\n- Storage/disk/SMART/capacity layer is closed with an explicit pve03 staging capacity monitor.\n- pve03 /mnt/staging was near WARN threshold at 77%, so a dedicated health file and timer were added.\n- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt.\n- Timer: homelab-pve03-staging-capacity-health.timer.\n- Final closure proof: 176_STORAGE_FINAL_CLOSURE_WITH_PVE03_MONITOR.txt.\n## SERVICE_DEPENDENCY_MAP_20260630\n\n### Ingress and DNS chain\n- Home DHCP DNS: [PRIVATE_IP] and [PRIVATE_IP].\n- dns1: CT110 / [PRIVATE_IP] / AdGuard Home.\n- dns2: CT111 / [PRIVATE_IP] / AdGuard Home.\n- unbound1: CT112 / [PRIVATE_IP]:5335.\n- unbound2: CT113 / [PRIVATE_IP]:5335.\n- dns1 upstream: [PRIVATE_IP]:5335.\n- dns2 upstream: [PRIVATE_IP]:5335.\n- Public ingress HTTP/HTTPS: router forwards TCP 80/443 to edge-vm [PRIVATE_IP].\n- Reverse proxy: NPMplus on edge-vm, container npmplus, host networking, admin bound to 127.0.0.1:81.\n- Edge runtime host: VM130 edge-vm, [PRIVATE_IP], Docker Compose projects count 43.\n- NPMplus proxy routes count: 47.\n- NPMplus certificates and proxy route source proof: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt.\n\n### Core public routes\n| Route | Upstream | Runtime owner | Backup/health evidence |\n|---|---|---|---|\n| nc.gram1.ru | http://[PRIVATE_IP]:11000 | VM150 Nextcloud AIO | VM150 vzdump, nextcloud restore proof, external canary |\n| uptime.gram1.ru | http://127.0.0.1:3001 | edge-vm / uptime-kuma | npmplus-kuma backup/restore, Kuma health |\n| gotify.gram1.ru | http://127.0.0.1:8082 | edge-vm / gotify | gotify health, alert-routing proof |\n| vault.gram1.ru | http://127.0.0.1:8083 | edge-vm / vaultwarden | vaultwarden backup/offhost/restore |\n| dockge.gram1.ru | http://127.0.0.1:5001 | edge-vm / dockge | stack inventory |\n| home.gram1.ru | http://127.0.0.1:3000 | edge-vm / homepage | homepage container, dashboard route |\n| git.gram1.ru | http://127.0.0.1:3002 | edge-vm / gitea | gitea backup/offhost/restore |\n| dozzle.gram1.ru | http://127.0.0.1:9999 | edge-vm / dozzle | docker stack inventory |\n| paper.gram1.ru | http://127.0.0.1:8010 | edge-vm / paperless | paperless backup/offhost/restore |\n| memos.gram1.ru | http://127.0.0.1:5230 | edge-vm / memos | memos backup/offhost/restore |\n| photos.gram1.ru | http://127.0.0.1:2283 | edge-vm / immich | immich media/full consistency proof |\n| auth.gram1.ru | http://127.0.0.1:9000 | edge-vm / authentik | authentik backup/offhost/restore |\n| backup.gram1.ru | http://[PRIVATE_IP]:9101 | pve01 / homelab-health-http | backup dashboard and health proofs |\n\n### VPN routes on wildcard certificate\n| Route | Upstream | Runtime owner |\n|---|---|---|\n| kuma.vpn.gram1.ru | http://127.0.0.1:3001 | uptime-kuma |\n| homepage.vpn.gram1.ru | http://127.0.0.1:3000 | homepage |\n| gitea.vpn.gram1.ru | http://127.0.0.1:3002 | gitea |\n| filebrowser.vpn.gram1.ru | http://127.0.0.1:8085 | filebrowser |\n| auth.vpn.gram1.ru | http://127.0.0.1:9000 | authentik |\n| actual-budget.vpn.gram1.ru | http://127.0.0.1:5006 | actual-budget |\n| grafana.vpn.gram1.ru | http://127.0.0.1:3003 | grafana |\n| homebox.vpn.gram1.ru | http://127.0.0.1:7745 | homebox |\n| mealie.vpn.gram1.ru | http://127.0.0.1:9001 | mealie |\n| n8n.vpn.gram1.ru | http://127.0.0.1:5678 | n8n |\n| netbox.vpn.gram1.ru | http://127.0.0.1:8000 | netbox |\n| node-red.vpn.gram1.ru | http://127.0.0.1:1880 | node-red |\n| beszel.vpn.gram1.ru | http://127.0.0.1:8090 | beszel |\n| it-tools.vpn.gram1.ru | http://127.0.0.1:8084 | it-tools |\n| karakeep.vpn.gram1.ru | http://127.0.0.1:3030 | karakeep |\n| linkding.vpn.gram1.ru | http://127.0.0.1:9091 | linkding |\n| minio.vpn.gram1.ru | http://127.0.0.1:9003 | minio |\n| ntfy.vpn.gram1.ru | http://127.0.0.1:8055 | ntfy |\n| searxng.vpn.gram1.ru | http://127.0.0.1:8888 | searxng |\n| syncthing.vpn.gram1.ru | http://127.0.0.1:8384 | syncthing |\n| alertmanager.vpn.gram1.ru | http://127.0.0.1:9093 | alertmanager |\n| prometheus.vpn.gram1.ru | http://127.0.0.1:9090 | prometheus |\n| homeassistant.vpn.gram1.ru | http://127.0.0.1:8123 | homeassistant |\n| npmplus.vpn.gram1.ru | https://[PRIVATE_IP]:81 | NPMplus admin, localhost-bound on edge |\n| proxmox.vpn.gram1.ru | https://[PRIVATE_IP]:8006 | pve01 Proxmox UI |\n| dns1.vpn.gram1.ru | http://[PRIVATE_IP]:8080 | dns1 AdGuard |\n| dns2.vpn.gram1.ru | http://[PRIVATE_IP]:8080 | dns2 AdGuard |\n| checks.vpn.gram1.ru | http://127.0.0.1:8015 | healthchecks |\n| vikunja.vpn.gram1.ru | http://127.0.0.1:8016 | vikunja |\n| bookstack.vpn.gram1.ru | http://127.0.0.1:8017 | bookstack |\n| pdf.vpn.gram1.ru | http://127.0.0.1:8018 | stirling-pdf |\n| jellyfin.vpn.gram1.ru | http://127.0.0.1:8019 | jellyfin |\n| audiobooks.vpn.gram1.ru | http://127.0.0.1:8020 | audiobookshelf |\n| books.vpn.gram1.ru | http://127.0.0.1:8021 | calibre-web |\n\n### Disabled / special routes\n- npm.gram1.ru exists in NPMplus but was observed disabled.\n- turn.gram1.ru is an AdGuard rewrite to [PRIVATE_IP] for Nextcloud Talk/TURN.\n- Router forwards TCP/UDP 3478 to Nextcloud [PRIVATE_IP].\n- Router forwards TCP/UDP 51820 for WireGuard on the Home bridge, not to edge-vm.\n\n### Critical dependency rules\n- If a public app is down, check in this order: DNS rewrite/upstream, NPMplus route/cert, upstream container/VM, app health file, backup/restore proof.\n- If a VPN route is down, check NetBird first, then NPMplus wildcard cert, then local upstream.\n- If DNS is broken, check dns1/dns2 AdGuard and unbound1/unbound2 before app containers.\n- If NPMplus is broken, use backup/restore evidence from npmplus-kuma-config and NPMplus DB backup.\n- If edge-vm is broken, VM130 full-image backup/offhost/restore proofs are authoritative.\n- If pve03 storage approaches WARN/CRIT, use pve03-staging-capacity health file.\n\n### Proof\n- Raw service dependency evidence index: 177_SERVICE_DEPENDENCY_MAP_RAW_EVIDENCE_INDEX.txt.\n- NPMplus route/cert rows: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt.\n- AdGuard/Unbound rewrites/upstreams: 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt.\n- Edge Docker container normalized map: 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt.\n- Proxmox VM/CT inventory: 121_PROXMOX_VM_CT_CONFIG_INVENTORY_REDACTED.txt.\n- Backup catalog: 141_REFERENCE_BACKUP_RESTORE_OFFHOST_CATALOG_BLOCK.txt.\n- Monitoring model: 147_REFERENCE_MONITORING_ALERTING_HEALTH_BLOCK.txt.\n\n## SERVICE_DEPENDENCY_MAP_FINAL_CLOSED_20260630\n\n- Service Dependency Map layer is closed.\n- Integrity proof: 180_SERVICE_DEPENDENCY_MAP_INTEGRITY_REVIEW.txt.\n- Secret scan proof: 179_SERVICE_DEPENDENCY_MAP_REFERENCE_PROOF_AND_SECRET_SCAN.txt.\n- Final closure proof: 181_SERVICE_DEPENDENCY_MAP_FINAL_CLOSURE.txt.\n## RUNBOOKS_RECOVERY_PROCEDURES_20260630\n\n### Universal operator preflight\n- Before every command, check /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md and the current 31_HOMELAB_REFERENCE.md.\n- Required visible markers: ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK.\n- If preflight fails, do not run the main action.\n- Do not use exit 1 in interactive SSH sessions.\n- Do not paste full terminal transcripts back into shell.\n- Do not print secrets.\n- Prefer short proof-producing commands over long nested quoting chains.\n\n### First triage order\n- Check current reference layer first.\n- Check latest proof file named by the relevant reference layer.\n- Check health files under /var/lib/homelab-health where applicable.\n- Check runtime state only after understanding the owning node, VM, container and proxy route.\n- Record every failed command or misleading proof in the error register before moving on.\n\n### Public application down\n- Start with Service Dependency Map.\n- Check DNS path: dns1/dns2 AdGuard, then unbound1/unbound2.\n- Check NPMplus route and certificate using NPMplus DB/proxy proof.\n- Check upstream app container or VM.\n- Check app-specific health, backup and restore proof.\n- Check external-canary and blackbox/Prometheus if route is public.\n- Do not change DNS, certificates or proxy routes without DB backup and proof.\n\n### DNS failure\n- dns1 is CT110 at [PRIVATE_IP].\n- dns2 is CT111 at [PRIVATE_IP].\n- unbound1 is CT112 at [PRIVATE_IP]:5335.\n- unbound2 is CT113 at [PRIVATE_IP]:5335.\n- AdGuard upstreams must point to local Unbound pair.\n- Rewrites include public routes for git, dozzle, paper, memos, photos, auth and backup to edge-vm.\n- turn.gram1.ru points to Nextcloud [PRIVATE_IP].\n- Use DNS/Ingress reference and proof files before editing configs.\n\n### Ingress / NPMplus failure\n- Edge VM is VM130 at [PRIVATE_IP].\n- NPMplus listens publicly on 80/443 and admin is bound to 127.0.0.1:81.\n- NPMplus DB is /opt/npmplus/npmplus/database.sqlite.\n- Before modifying certificates or proxy rows, create a DB backup.\n- Cloudflare token values must never be printed.\n- Use npmplus-kuma-config backup/offhost/restore proofs for recovery.\n\n### Edge VM failure\n- VM130 is the Docker runtime host.\n- First check Proxmox VM state and pve03 storage.\n- Then use VM130 full-image vzdump/offhost/restore proofs.\n- Current authoritative edge-vm backup/offhost/restore evidence is in backup catalog and VM130 proofs.\n- If restoring, verify exact archive, checksum and VMA/ZSTD integrity before booting replacement.\n\n### Proxmox / VM / CT restore\n- Primary nightly job: homelab-nightly-all at 03:30, zstd, snapshot.\n- Included VMIDs: 110,111,112,113,130,150,160.\n- Use backup catalog for latest known backup/offhost/restore evidence.\n- Do not infer offhost success from failed rsync.\n- Verify exact artifact, size and checksum where available.\n- For VM130 and VM160, use their dedicated closure proofs.\n\n### Monitoring / alerting failure\n- Prometheus is on edge-vm at 127.0.0.1:9090.\n- Alertmanager is on 127.0.0.1:9093.\n- Loki is on 127.0.0.1:3100.\n- Uptime Kuma is on 127.0.0.1:3001.\n- Gotify is on 127.0.0.1:8082.\n- ntfy is on 127.0.0.1:8055.\n- Healthchecks is on 127.0.0.1:8015.\n- Prometheus proof 153 is invalid and must not be used.\n- Use proof 154 and final closure 157 for Prometheus settled target state.\n\n### Backup dashboard / SLO interpretation\n- Runtime dashboard OK means current operational checks are green.\n- SLO backlog WARN means coverage improvement backlog, not necessarily runtime outage.\n- Do not merge SLO backlog WARN into runtime blockers unless explicitly promoted.\n- Use slo-coverage-backlog-index for backlog status.\n\n### Storage / capacity event\n- pve03 staging is monitored separately because it was observed at 77%.\n- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt.\n- WARN threshold: 80%.\n- CRIT threshold: 90%.\n- Timer: homelab-pve03-staging-capacity-health.timer.\n- Before cleanup, check retention policy and backup/offhost proof.\n- Do not use broad Docker prune by default.\n\n### Security / secret incident\n- Stop printing values immediately.\n- Identify whether the leak is value, file path, or false-positive text.\n- Rotate affected token/password if a value was exposed.\n- Create backup before DB/config mutation.\n- Re-run strict secret scan after rotation.\n- Record proof files and external revocation confirmation where applicable.\n- Cloudflare token and XenForo SMTP rotations already have closure proofs.\n\n### Forum / CodeVipe incident\n- forum-prod is VM160 at [PRIVATE_IP].\n- Access path is through pve02 using [SENSITIVE_PATH]\n- XenForo path: /var/www/codevipe/public.\n- SMTP host: mail.pvepro.ru on port 587 with STARTTLS semantics.\n- Do not enable smtpSsl=true blindly for port 587.\n- Do not use fragile nested PHP/base64 SMTP checkers.\n- Use XenForo SMTP rotation and auth proof files for current mail state.\n\n### Final evidence rules\n- Every remediation gets a numbered proof file.\n- Every reference block gets a proof and secret scan file.\n- Invalid proof files must be explicitly superseded, not silently ignored.\n- Reference closure requires integrity scan, secret scan and required-heading checks.\n## HOMELAB_REFERENCE_FINAL_OPERATOR_STATUS_20260630\n\n### Status\n- Current homelab reference is complete for the audited local infrastructure scope.\n- Final quality precheck passed.\n- Required headings are present.\n- Bad terminal/log marker scan is clean.\n- Strict secret scan is clean.\n- This is a reference/operator-status closure, not an archive/export.\n\n### Closed layers\n- Critical runtime tails closure.\n- Proxmox cluster, node, storage and VM/CT inventory.\n- DNS, ingress, NPMplus certificates and AdGuard/Unbound model.\n- Edge Docker stacks and container map.\n- Backup, restore, offhost and cloud backup catalog.\n- Monitoring, alerting, health dashboard and Prometheus correction.\n- Security, access and secrets operating model.\n- Storage, disk, SMART and capacity model with pve03 staging monitor.\n- Service Dependency Map.\n- Runbooks and recovery procedures.\n\n### Important superseded/invalid proofs\n- 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt is invalid and must not be used.\n- 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt is not sufficient by itself because symlink mode showed 777.\n- Use 154 for Prometheus settled targets and 163 for symlink-aware SSH target permissions.\n\n### Current authoritative final proof set\n- 185_HOMELAB_REFERENCE_FINAL_QUALITY_SCAN_PRECHECK.txt.\n- 184_RUNBOOKS_RECOVERY_REFERENCE_PROOF_AND_SECRET_SCAN.txt.\n- 181_SERVICE_DEPENDENCY_MAP_FINAL_CLOSURE.txt.\n- 176_STORAGE_FINAL_CLOSURE_WITH_PVE03_MONITOR.txt.\n- 165_SECURITY_ACCESS_SECRETS_FINAL_PROOF_AND_SECRET_SCAN.txt.\n- 157_MONITORING_PROMETHEUS_FINAL_CLOSURE.txt.\n- 142_BACKUP_RESTORE_OFFHOST_REFERENCE_PROOF_AND_SECRET_SCAN.txt.\n- 136_EDGE_DOCKER_STACKS_REFERENCE_PROOF_AND_SECRET_SCAN.txt.\n- 132_DNS_INGRESS_REFERENCE_PROOF_AND_SECRET_SCAN.txt.\n\n### Operator rule\n- Before every future command, continue checking both the error register and this reference file.\n- Required visible markers remain ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK.\n## HOMELAB_REFERENCE_DEEP_AUDIT_CLOSURE_20260630\n\n### Deep audit result\n- Error register and reference consistency audit passed.\n- Final proof files unresolved-marker audit passed.\n- Reference proof-link audit passed.\n- All referenced proof files exist.\n- No unresolved final REVIEW_REQUIRED, PROOF_MISSING, HEADING_MISSING, BAD_MARKER, STRICT_SECRET, RULE_CHECK_FAIL, SyntaxError or PERM_BAD markers remained in final proof set.\n- Invalid/superseded proofs are explicitly documented and must not be used as authoritative closure evidence.\n\n### Authoritative deep audit proofs\n- 189_DEEP_ERROR_REGISTER_REFERENCE_CONSISTENCY_AUDIT.txt.\n- 190_DEEP_PROOF_FILES_UNRESOLVED_MARKER_AUDIT.txt.\n- 191_DEEP_REFERENCE_PROOF_LINK_AUDIT.txt.\n\n### Scope statement\n- This closes the current audited local homelab reference scope.\n- External systems can still receive separate dedicated passports if the scope is expanded later.\n\n## NEXTCLOUD_VM150_MAIL_CLOUD_BACKUP_20260630\n- VM150 Nextcloud Mail-cloud backup is installed on pve01.\n- Timer: homelab-mail-cloud-nextcloud-vm-upload.timer.\n- Health file: /var/lib/homelab-health/mail-cloud-nextcloud-vm.txt.\n- Current proof health: STATUS=OK, MODE=recurring-chunked, PARTS=14, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no.\n- Proof: 192_NEXTCLOUD_VM150_MAIL_CLOUD_CHUNKED_PROOF.txt.\n\n## ROUTER_RECURRING_BACKUP_BLOCKER_20260630\n- Router recurring backup from pve01 is not installed yet.\n- Reason: pve01 cannot reach router management ports 2222, 5080, 5083 or 2323 on [PRIVATE_IP].\n- Current router backup state remains reference/self-test based until router management access is allowed or startup-config is exported by another approved path.\n- Do not run bash/POSIX automation against Netcraze CLI; use router CLI syntax only.\n- Proof: 195_ROUTER_RECURRING_BACKUP_BLOCKED_BY_MGMT_ACL_PROOF.txt.\n\n## EDGE_GAP_RESTORE_DRYRUN_CLOSURE_20260630\n- Restore dry-run gaps closed for Actual Budget, Home Assistant, Karakeep, Linkding, Mealie, n8n and Observability config.\n- Production data was not restored or modified; checks used temporary extraction, SHA verification, tar listing/extraction and SQLite integrity where applicable.\n- Current observed restore health: all 7 STATUS=OK with RESTORE_ATTEMPTED=YES and SECRET_PRINTED=no.\n- Proof: 197_EDGE_GAP_RESTORE_DRYRUN_CLOSURE_PROOF.txt.\n\n## P2_SMALL_STACKS_BACKUP_RESTORE_CLOSURE_20260630\n- P2 small-stack backup and restore dry-run is installed on edge-vm.\n- Covered paths: Syncthing config, ntfy cache/data, MinIO data, Homebox data and Node-RED data.\n- Timer: homelab-p2-small-stacks-backup-restore.timer.\n- Current observed health: STATUS=OK, BACKUP_PATH_COUNT=6, RESTORE_ATTEMPTED=YES, SQLITE_INTEGRITY_OK=yes, SECRET_PRINTED=no.\n- Proof: 199_P2_SMALL_STACKS_BACKUP_RESTORE_CLOSURE_PROOF.txt.\n\n## BACKUP_CLOUD_AND_RESTORE_FINAL_CLOSURE_20260630\n- Backup/cloud/restore improvement pass is closed with runtime dashboard STATUS=OK and NOT_OK=0.\n- VM150 Nextcloud now has Mail-cloud chunked backup with download verification.\n- Restore dry-run gaps closed for Actual Budget, Home Assistant, Karakeep, Linkding, Mealie, n8n and Observability config.\n- P2 small stacks now have backup and restore dry-run coverage for Syncthing config, ntfy, MinIO, Homebox and Node-RED.\n- Router recurring backup remains blocked by management ACL and is documented as blocker, not silently closed.\n- Proof: 201_BACKUP_CLOUD_AND_RESTORE_FINAL_CLOSURE_PROOF.txt.\n\n## ROUTER_STARTUP_CONFIG_MAIL_CLOUD_MANUAL_20260630\n- Router startup-config manual backup is stored in Mail-cloud crypt remote.\n- Source file content is not printed in proofs or reference.\n- Health file: /var/lib/homelab-health/router-startup-config-mail-cloud.txt.\n- Current observed health: STATUS=OK, UPLOAD_OK=yes, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no.\n- Recurring router backup is still blocked until pve01 can reach router management ports.\n- Proof: 203_ROUTER_STARTUP_CONFIG_MAIL_CLOUD_MANUAL_PROOF.txt.\n\n## ROUTER_RUNNING_CONFIG_MAIL_CLOUD_RECURRING_20260630\n- Router running-config recurring Mail-cloud backup is installed on pve01.\n- Timer: homelab-router-running-config-mail-cloud.timer.\n- Health file: /var/lib/homelab-health/router-running-config-mail-cloud.txt.\n- Current observed health: STATUS=OK, SSH_RC=0, LOOKS_CONFIG=yes, UPLOAD_OK=yes, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no.\n- Manual startup-config Mail-cloud backup also exists as separate proof.\n- Proof: 205_ROUTER_RUNNING_CONFIG_MAIL_CLOUD_RECURRING_PROOF.txt.\n\n## POST_BACKUP_PASS_SLO_RECONCILIATION_20260630\n- Post backup/cloud/restore pass SLO reconciliation is closed.\n- Runtime backup dashboard remains STATUS=OK with NOT_OK=0.\n- Restore index remains STATUS=OK with RESTORE_PROOFS=16/16.\n- VM150 Mail-cloud backup, router manual/recurring Mail-cloud backups, P2 small-stacks and 7 edge restore dry-runs are all confirmed OK.\n- Existing backup-restore-coverage-slo.txt is historical model output from 20260625 and does not represent current runtime blocker state.\n- Current reconciliation health: STATUS=OK, RUNTIME_BLOCKER=NO, SECRET_PRINTED=no.\n- Proof: 208_POST_BACKUP_PASS_SLO_RECONCILIATION_PROOF.txt.\n\n## POST_BACKUP_PASS_ALERTING_20260630\n- Prometheus alerting for post-backup-pass health files is installed and loaded.\n- Rule file: /opt/stacks/observability-lite/prometheus/rules/homelab-post-backup-pass-alerts.yml.\n- Alerts: HomelabPostBackupPassHealthNotOk and HomelabPostBackupPassHealthStale.\n- Covered checks include VM150 Mail-cloud, router startup/running config Mail-cloud, P2 small-stacks and seven restore dry-run health files.\n- Current proof confirms rule validation, Prometheus API visibility and up targets.\n- Proof: 210_POST_BACKUP_PASS_ALERTING_PROOF.txt.\n\n## POST_BACKUP_PASS_ALERTS_NOT_FIRING_20260630\n- Post-backup-pass Prometheus alert rules are loaded and currently not firing.\n- Corrected proof uses direct PromQL ALERTS query for firing state, not grep over rule names.\n- Alerts checked: HomelabPostBackupPassHealthNotOk and HomelabPostBackupPassHealthStale.\n- Proof: 213_POST_BACKUP_PASS_ALERTS_NOT_FIRING_CORRECTED_PROOF.txt.\n\n## MAIL_CLOUD_CAPACITY_RETENTION_20260630\n- Mail-cloud capacity and backup directory inventory was checked after post-backup-pass closure.\n- VM150 Nextcloud, router startup/running config, P0 critical and edge-vm backup directories are present on crypt remotes.\n- Observed capacity is sufficient: largest used remote was pve03-mail-03-crypt around 65 GiB of 1 TiB.\n- Retention shape observed: P0 critical has 10 visible dirs with keep 14, edge-vm has 3 visible dirs with keep 4, new VM150/router jobs have initial dirs.\n- Proof: 216_MAIL_CLOUD_CAPACITY_RETENTION_PROOF.txt.\n\n## ROUTER_BACKUP_SECRET_PERMISSION_20260630\n- Router recurring backup uses a dedicated routerbackup credential file on pve01.\n- Secret file path: /var/lib/homelab-private/secrets/routerbackup.pass.\n- File content must never be printed; only mode/owner/size may be checked.\n- Current observed permission target: root-owned mode 600.\n- Proof: 218_ROUTER_BACKUP_SECRET_PERMISSION_PROOF.txt.\n\n## NEW_BACKUP_UNITS_SYSTEMD_INVENTORY_20260630\n- New backup/restore systemd units and timers were inventoried after post-backup-pass closure.\n- pve01 units checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup.\n- edge-vm units checked: P2 small-stacks backup/restore, backup dashboard and restore drill index.\n- Proof records LoadState, ActiveState, UnitFileState and failed-unit counts for pve01 and edge-vm.\n- Proof: 222_NEW_BACKUP_UNITS_SYSTEMD_INVENTORY_PROOF.txt.\n\n## POST_BACKUP_PASS_PROM_HEALTH_COVERAGE_20260630\n- Prometheus health coverage was checked for the post-backup-pass checks.\n- Covered checks: VM150 Mail-cloud, router startup/running config, P2 small-stacks and seven restore dry-runs.\n- Proof records health status and age from Prometheus without printing credential values.\n- Proof: 224_POST_BACKUP_PASS_PROM_HEALTH_COVERAGE_PROOF.txt.\n\n## POST_BACKUP_PASS_AUDIT_INDEX_20260630\n- Audit proof manifest was generated for post-backup-pass evidence files 192-225.\n- Manifest records file names, sha256 hashes, count and missing-number check.\n- Proof: 226_POST_BACKUP_PASS_AUDIT_INDEX_PROOF.txt.\n\n## POST_BACKUP_PASS_EXTENDED_FINAL_SNAPSHOT_20260630\n- Extended post-backup-pass final snapshot was created after capacity, retention, unit inventory, Prometheus coverage and audit index checks.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-227.\n- Proof: 228_POST_BACKUP_PASS_EXTENDED_FINAL_SNAPSHOT_PROOF.txt.\n\n## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_229_20260630\n- Final audit manifest was generated after the extended final snapshot.\n- Manifest covers proof numbers 192-229, including snapshot and reference-scan proofs.\n- Manifest records file names, sha256 hashes, count and missing-number check.\n- Proof: 230_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_229_PROOF.txt.\n\n## NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_20260630\n- New backup/restore unit files and executable script paths were inventoried and hashed.\n- pve01 services checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup.\n- edge-vm services checked: P2 small-stacks backup/restore, backup dashboard and restore drill index.\n- Proof records FragmentPath, ExecStart path, mode/owner/size and sha256 hashes without printing script contents.\n- Proof: 232_NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_PROOF.txt.\n\n## NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_CORRECTED_20260630\n- Corrected integrity proof was generated because one edge-vm executable required sudo for sha256sum.\n- Corrected proof records hashes for pve01 and edge-vm unit fragments and executable scripts without printing script contents.\n- Proof: 234_NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_CORRECTED_PROOF.txt.\n\n## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_235_20260630\n- Final audit manifest was regenerated after corrected unit/script integrity proof.\n- Manifest covers proof numbers 192-235, including corrected integrity proof 234/235.\n- Manifest records file names, sha256 hashes, count and missing-number check.\n- Proof: 236_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_235_PROOF.txt.\n\n## NEW_BACKUP_TIMERS_INTEGRITY_20260630\n- New backup/restore timer unit files were inventoried and hashed.\n- pve01 timers checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup.\n- edge-vm timers checked: P2 small-stacks backup/restore, backup dashboard and restore drill index.\n- Proof records FragmentPath, mode/owner/size, sha256 hashes and next-elapse metadata without printing sensitive values.\n- Proof: 238_NEW_BACKUP_TIMERS_INTEGRITY_PROOF.txt.\n\n## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_239_20260630\n- Final audit manifest was regenerated after timer integrity proof.\n- Manifest covers proof numbers 192-239 and records file names, sha256 hashes, count and missing-number check.\n- Proof: 240_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_239_PROOF.txt.\n\n## POST_BACKUP_PASS_FINAL_SNAPSHOT_AFTER_TIMERS_20260630\n- Final snapshot was created after timer integrity and final audit manifest 192-239.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-241.\n- Proof: 242_POST_BACKUP_PASS_FINAL_SNAPSHOT_AFTER_TIMERS_PROOF.txt.\n\n## ASSISTANT_COMMAND_BATCHING_RULE_20260630\n- Operator preference: combine related homelab commands into fewer package-style runs to reduce slow repetitive manual work.\n- Applies to proof generation, reference updates, validation scans, hash/index refreshes and snapshot steps when safe.\n- Mandatory preflight and sensitive-output hygiene still take priority.\n\n## POST_BACKUP_PASS_AUDIT_INDEX_AFTER_BATCHING_RULE_192_245_20260630\n- Audit manifest was regenerated after assistant command batching rule was recorded.\n- Manifest covers proof numbers 192-245 and records file names, sha256 hashes, count and missing-number check.\n- Proof: 246_POST_BACKUP_PASS_AUDIT_INDEX_AFTER_BATCHING_RULE_192_245_PROOF.txt.\n\n## POST_BACKUP_PASS_SNAPSHOT_AFTER_BATCHING_RULE_20260630\n- Snapshot was created after assistant command batching rule and audit manifest 192-245.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-247.\n- Proof: 248_POST_BACKUP_PASS_SNAPSHOT_AFTER_BATCHING_RULE_PROOF.txt.\n\n## ALERTMANAGER_ROUTING_AND_CONFIG_20260630\n- Alertmanager routing/config health was checked after post-backup-pass alert rules were installed.\n- Proof records container presence, config validation, readiness endpoint and sanitized route/receiver metadata.\n- Sensitive receiver values and URLs are intentionally not printed.\n- Proof: 250_ALERTMANAGER_ROUTING_AND_CONFIG_PROOF.txt.\n\n## AUDIT_INDEX_192_251_20260630\n- Audit manifest was regenerated after Alertmanager routing/config proof.\n- Manifest covers proof numbers 192-251 with count and missing-number check.\n- Proof: 252_AUDIT_INDEX_192_251_PROOF.txt.\n\n## RCLONE_CONFIG_PERMISSION_AND_REMOTE_INVENTORY_20260630\n- rclone config permissions and crypt remote inventory were checked without printing config contents.\n- Proof records config owner/mode/size, remote names and capacity summaries for active Mail-cloud crypt remotes.\n- Proof: 254_RCLONE_CONFIG_PERMISSION_AND_REMOTE_INVENTORY_PROOF.txt.\n\n## AUDIT_INDEX_192_255_20260630\n- Audit manifest was regenerated after rclone config/remote inventory proof.\n- Manifest covers proof numbers 192-255 with count and missing-number check.\n- Proof: 256_AUDIT_INDEX_192_255_PROOF.txt.\n\n## CURRENT_OPERATIONAL_ROLLUP_20260630\n- Current operational rollup was captured after rclone config/remote inventory proof.\n- Rollup records failed-unit counts, next timers, Mail-cloud capacity, Prometheus target count, post-backup health status/age, post-backup firing alerts and Alertmanager readiness.\n- Proof: 258_CURRENT_OPERATIONAL_ROLLUP_PROOF.txt.\n\n## AUDIT_INDEX_192_259_20260630\n- Audit manifest was regenerated after current operational rollup.\n- Manifest covers proof numbers 192-259 with count and missing-number check.\n- Proof: 260_AUDIT_INDEX_192_259_PROOF.txt.\n\n## SNAPSHOT_AFTER_CURRENT_OPERATIONAL_ROLLUP_20260630\n- Snapshot was created after current operational rollup and audit index 192-259.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-261.\n- Proof: 262_SNAPSHOT_AFTER_CURRENT_OPERATIONAL_ROLLUP_PROOF.txt.\n\n## AUDIT_INDEX_192_263_20260630\n- Audit manifest was regenerated after snapshot following current operational rollup.\n- Manifest covers proof numbers 192-263 with count and missing-number check.\n- Proof: 264_AUDIT_INDEX_192_263_PROOF.txt.\n\n## NEW_BACKUP_SERVICE_RUNTIME_RESULT_20260630\n- Runtime result metadata was captured for new backup/restore service units.\n- pve01 services checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup.\n- edge-vm services checked: P2 small-stacks backup/restore, backup dashboard and restore drill index.\n- Proof records LoadState, ActiveState, Result, ExecMainStatus and restart counters without printing sensitive values.\n- Proof: 266_NEW_BACKUP_SERVICE_RUNTIME_RESULT_PROOF.txt.\n\n## AUDIT_INDEX_192_267_20260630\n- Audit manifest was regenerated after new backup service runtime-result proof.\n- Manifest covers proof numbers 192-267 with count and missing-number check.\n- Proof: 268_AUDIT_INDEX_192_267_PROOF.txt.\n\n## NEW_BACKUP_UNITS_JOURNAL_ERROR_SCAN_20260630\n- Journal warning/error counters were captured for new backup/restore units and timers over the last 24 hours.\n- Proof records only counts, not journal message bodies, to avoid sensitive-output risk.\n- Proof: 270_NEW_BACKUP_UNITS_JOURNAL_ERROR_SCAN_PROOF.txt.\n\n## AUDIT_INDEX_192_271_20260630\n- Audit manifest was regenerated after journal error scan proof.\n- Manifest covers proof numbers 192-271 with count and missing-number check.\n- Proof: 272_AUDIT_INDEX_192_271_PROOF.txt.\n\n## NEW_BACKUP_UNITS_JOURNAL_ERROR_TRIAGE_20260630\n- Journal warning/error triage was captured after warning/error counters were non-zero.\n- Proof records per-unit warning/error counts and redacted journal fragments.\n- Proof: 274_NEW_BACKUP_UNITS_JOURNAL_ERROR_TRIAGE_PROOF.txt.\n\n## AUDIT_INDEX_192_275_20260630\n- Audit manifest was regenerated after journal error triage proof.\n- Manifest covers proof numbers 192-275 with count and missing-number check.\n- Proof: 276_AUDIT_INDEX_192_275_PROOF.txt.\n\n## VM150_MAIL_CLOUD_JOURNAL_NOISE_CLASSIFICATION_20260630\n- VM150 Mail-cloud upload journal warnings/errors were classified after journal triage found non-zero counters.\n- Current service result, Prometheus health and firing-alert status are used to decide whether journal noise is blocking.\n- Proof includes redacted journal indicators only, not secrets.\n- Proof: 278_VM150_MAIL_CLOUD_JOURNAL_NOISE_CLASSIFICATION_PROOF.txt.\n\n## AUDIT_INDEX_192_279_20260630\n- Audit manifest was regenerated after VM150 Mail-cloud journal-noise classification.\n- Manifest covers proof numbers 192-279 with count and missing-number check.\n- Proof: 280_AUDIT_INDEX_192_279_PROOF.txt.\n\n## SNAPSHOT_AFTER_VM150_JOURNAL_NOISE_CLASSIFICATION_20260630\n- Snapshot was created after VM150 Mail-cloud journal noise was classified as non-blocking historical noise.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-281.\n- Proof: 282_SNAPSHOT_AFTER_VM150_JOURNAL_NOISE_CLASSIFICATION_PROOF.txt.\n\n## AUDIT_INDEX_192_283_20260630\n- Audit manifest was regenerated after snapshot following VM150 journal-noise classification.\n- Manifest covers proof numbers 192-283 with count and missing-number check.\n- Proof: 284_AUDIT_INDEX_192_283_PROOF.txt.\n\n## POST_BACKUP_PASS_CLOSURE_SUMMARY_20260630\n- Post-backup-pass closure summary was generated after VM150 journal-noise classification.\n- Summary checks backup SLO closure, alert rules, firing-alert status, Prometheus health coverage, failed-unit state, Alertmanager readiness, rclone config hygiene, journal-noise classification and audit index completeness.\n- Proof: 286_POST_BACKUP_PASS_CLOSURE_SUMMARY_PROOF.txt.\n\n## SNAPSHOT_AFTER_POST_BACKUP_PASS_CLOSURE_SUMMARY_20260630\n- Snapshot was created after post-backup-pass closure summary.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range: 192-287.\n- Proof: 288_SNAPSHOT_AFTER_POST_BACKUP_PASS_CLOSURE_SUMMARY_PROOF.txt.\n\n## AUDIT_INDEX_192_289_20260630\n- Audit manifest was regenerated after post-backup-pass closure summary snapshot.\n- Manifest covers proof numbers 192-289 with count and missing-number check.\n- Proof: 290_AUDIT_INDEX_192_289_PROOF.txt.\n\n## TOMORROW_FIRST_RUN_VERIFICATION_PLAN_20260630\n- First scheduled-run verification should be done after 2026-07-01 08:15 MSK.\n- Verify P2 small-stacks, router running-config Mail-cloud and VM150 Nextcloud Mail-cloud first automatic runs.\n- Also verify service results, health status/age, Mail-cloud directories, dashboard, restore index, Prometheus firing alerts and Alertmanager readiness.\n- Proof: 292_TOMORROW_FIRST_RUN_VERIFICATION_PLAN_PROOF.txt.\n\n## AUDIT_INDEX_192_293_20260630\n- Audit manifest was regenerated after tomorrow first-run verification plan.\n- Manifest covers proof numbers 192-293 with count and missing-number check.\n- Proof: 294_AUDIT_INDEX_192_293_PROOF.txt.\n\n## END_OF_DAY_POST_BACKUP_PASS_SNAPSHOT_20260630\n- End-of-day snapshot was created after post-backup-pass closure summary and tomorrow first-run verification plan.\n- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes.\n- Covered proof range before this snapshot: 192-295.\n- Proof: 296_END_OF_DAY_POST_BACKUP_PASS_SNAPSHOT_PROOF.txt.\n\n## AUDIT_INDEX_192_297_20260630\n- Final end-of-day audit manifest was generated after post-backup-pass snapshot.\n- Manifest covers proof numbers 192-297 with count and missing-number check.\n- Proof: 298_AUDIT_INDEX_192_297_PROOF.txt.\n\n## HOME_PORTAL_DISCOVERY_20260630\n- Home portal discovery started for https://home.gram1.ru/.\n- Goal: make the portal show and open all web services, including external service cards such as NetBird and mail.\n- Discovery proof records DNS, HTTPS/TLS headers, reverse-proxy candidates, compose files, domain references and homepage config candidates without printing secrets.\n- Proof: 300_HOME_PORTAL_DISCOVERY_PROOF.txt.\n\n## HOME_PORTAL_CONFIG_AND_SERVICE_INVENTORY_20260630\n- Home portal config and service inventory was collected for https://home.gram1.ru/.\n- Inventory records portal containers, config files, redacted card lines, compose web hints, domain references and common subdomain probes.\n- Proof: 302_HOME_PORTAL_CONFIG_AND_SERVICE_INVENTORY_PROOF.txt.\n\n## HOME_PORTAL_GAP_ANALYSIS_20260630\n- Home portal gap analysis was collected for gethomepage/homepage behind npmplus.\n- Proof records Homepage config hashes, redacted current cards/bookmarks, current URLs, npmplus route lines, local web-port probes and known service container candidates.\n- Proof: 304_HOME_PORTAL_GAP_ANALYSIS_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_305_20260630\n- Home portal audit manifest was generated for proof numbers 300-305.\n- Proof: 306_HOME_PORTAL_AUDIT_INDEX_300_305_PROOF.txt.\n\n## HOME_PORTAL_CARD_AND_API_ERROR_ANALYSIS_20260630\n- Home portal card/API-error analysis was collected before editing Homepage.\n- Proof records current card URLs/titles, redacted widget config, redacted Homepage log errors, NPMPlus internal route files/routes and candidate public/VPN cards.\n- Proof: 308_HOME_PORTAL_CARD_AND_API_ERROR_ANALYSIS_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_309_20260630\n- Home portal audit manifest was regenerated after card/API-error analysis.\n- Manifest covers proof numbers 300-309 with count and missing-number check.\n- Proof: 310_HOME_PORTAL_AUDIT_INDEX_300_309_PROOF.txt.\n\n## HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_APPLY_20260630\n- Homepage services.yaml was backed up, duplicate VPN cards were removed where a non-VPN card existed, and missing web cards for NetBird, Mail and Webmail were added without VPN suffix in the card names.\n- SMTP/IMAP were intentionally not added as cards because they are protocol endpoints, not web UI cards.\n- Homepage container was restarted and portal/card URLs were checked.\n- Proof: 312_HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_APPLY_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_313_20260630\n- Home portal audit manifest was regenerated after duplicate-card cleanup and external-card addition.\n- Manifest covers proof numbers 300-313 with count and missing-number check.\n- Proof: 314_HOME_PORTAL_AUDIT_INDEX_300_313_PROOF.txt.\n\n## HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_CORRECTED_APPLY_20260630\n- Corrected Homepage apply removed duplicate VPN cards where a non-VPN card existed and added NetBird, Mail and Webmail cards with non-VPN names.\n- The added card URLs use currently reachable vpn.gram1.ru web endpoints because public netbird/mail/webmail hosts did not resolve/open during analysis.\n- SMTP/IMAP were not added as cards because they are protocol endpoints, not web UIs.\n- Proof: 316_HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_CORRECTED_APPLY_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_317_20260630\n- Home portal audit manifest was regenerated after corrected duplicate-card cleanup and external-card addition.\n- Manifest covers proof numbers 300-317 with count and missing-number check.\n- Proof: 318_HOME_PORTAL_AUDIT_INDEX_300_317_PROOF.txt.\n\n## HOMELAB_COMMAND_SAFETY_HARDENING_20260630\n- Command safety rule strengthened after the home portal base64 apply failure.\n- Opaque generated payloads are not acceptable for homelab changes; commands must be transparent, bounded, and validated before modification.\n- Success requires content-specific checks in addition to service/runtime checks.\n- Proof: 320_HOMELAB_COMMAND_SAFETY_HARDENING_RULE_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_321_20260630\n- Home portal audit manifest was regenerated after command-safety hardening rule.\n- Manifest covers proof numbers 300-321 with count and missing-number check.\n- Proof: 322_HOME_PORTAL_AUDIT_INDEX_300_321_PROOF.txt.\n\n## HOME_PORTAL_POST_APPLY_VALIDATION_AND_API_TRIAGE_20260630\n- Home portal was validated after duplicate cleanup and external-card addition.\n- Proof checks portal HTTP status, Homepage container state, required cards, duplicate VPN card pairs, current card URLs, redacted API-error logs and widget configuration.\n- Proof: 324_HOME_PORTAL_POST_APPLY_VALIDATION_AND_API_TRIAGE_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_325_20260630\n- Home portal audit manifest was regenerated after post-apply validation and API triage.\n- Manifest covers proof numbers 300-325 with count and missing-number check.\n- Proof: 326_HOME_PORTAL_AUDIT_INDEX_300_325_PROOF.txt.\n\n## HOME_PORTAL_API_ERROR_ROOT_CAUSE_20260630\n- Homepage API error root-cause analysis was collected after the UI showed API errors.\n- Current evidence points to the Open-Meteo/weather widget timing out from Homepage, separate from service cards.\n- HTTP 200 for the portal is not sufficient for UI closure when Homepage logs still contain API errors.\n- Proof: 328_HOME_PORTAL_API_ERROR_ROOT_CAUSE_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_329_20260630\n- Home portal audit manifest was regenerated after API-error root-cause analysis.\n- Manifest covers proof numbers 300-329 with count and missing-number check.\n- Proof: 330_HOME_PORTAL_AUDIT_INDEX_300_329_PROOF.txt.\n\n## HOME_PORTAL_DISABLE_OPENMETEO_WIDGET_20260630\n- Open-Meteo/weather widget was disabled after root-cause analysis showed Homepage API errors from api.open-meteo.com timeouts.\n- This keeps the portal focused on service-launcher cards and prevents external weather API failures from causing UI API errors.\n- Portal closure now requires current Homepage API-error logs to be zero after restart/reload.\n- Proof: 332_HOME_PORTAL_DISABLE_OPENMETEO_WIDGET_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_333_20260630\n- Home portal audit manifest was regenerated after disabling Open-Meteo/weather widget.\n- Manifest covers proof numbers 300-333 with count and missing-number check.\n- Proof: 334_HOME_PORTAL_AUDIT_INDEX_300_333_PROOF.txt.\n\n## HOME_PORTAL_LINK_OPEN_AUDIT_20260630\n- Home portal card links were audited after Open-Meteo/weather widget was disabled.\n- Audit extracts current Homepage href URLs and checks whether each opens with an acceptable HTTP status.\n- Acceptable statuses include success, redirects and auth-gated 401/403 pages because those still indicate a reachable web service.\n- Proof: 336_HOME_PORTAL_LINK_OPEN_AUDIT_PROOF.txt.\n\n## HOME_PORTAL_AUDIT_INDEX_300_337_20260630\n- Home portal audit manifest was regenerated after link-open audit.\n- Manifest covers proof numbers 300-337 with count and missing-number check.\n- Proof: 338_HOME_PORTAL_AUDIT_INDEX_300_337_PROOF.txt.\n\n## HOME_PORTAL_EXPLICIT_VPN_DUPLICATE_CLEANUP_CORRECTED_20260630\n- Explicit VPN duplicate cards were removed from Homepage with a short transparent perl edit: Homepage VPN, Authentik VPN, Gitea VPN and Uptime Kuma VPN.\n- Old router URL [PRIVATE_IP] was replaced with [PRIVATE_IP] where present.\n- Proof: 342_HOME_PORTAL_EXPLICIT_VPN_DUPLICATE_CLEANUP_CORRECTED_PROOF.txt.\n\n## HOME_PORTAL_REMAINING_DUPLICATE_OCCURRENCE_ANALYSIS_20260630\n- Remaining Homepage duplicate/VPN/router occurrences were inspected after corrected cleanup still reported REVIEW.\n- Proof records exact redacted file/line occurrences before another edit.\n- Proof: 344_HOME_PORTAL_REMAINING_DUPLICATE_OCCURRENCE_ANALYSIS_PROOF.txt.\n\n## HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RECHECK_20260630\n- Homepage duplicate cleanup was rechecked against active config files only, excluding logs and backup files.\n- Previous broad grep included historical logs/backups and therefore overstated remaining active duplicates.\n- Proof: 346_HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RECHECK_PROOF.txt.\n\n## HOME_PORTAL_ACTIVE_VPN_DUPLICATES_FINAL_CLEANUP_20260630\n- Active Homepage services/bookmarks files were cleaned from semantic VPN duplicate cards and old router URL references.\n- Removed targets: Homepage VPN, Authentik VPN, Gitea VPN and Uptime Kuma VPN, plus matching vpn.gram1.ru duplicate URLs.\n- Proof: 350_HOME_PORTAL_ACTIVE_VPN_DUPLICATES_FINAL_CLEANUP_PROOF.txt.\n\n## HOME_PORTAL_NPMPLUS_DEFAULT_ROUTE_ANALYSIS_20260630\n- Home portal NetBird/Mail/Webmail cards were investigated after user saw the NPMPlus default page from client [PRIVATE_IP].\n- Analysis checks URL body fingerprints, NPMPlus route config matches, service container candidates and Homepage card lines.\n- Proof: 356_HOME_PORTAL_NPMPLUS_DEFAULT_ROUTE_ANALYSIS_PROOF.txt.\n\n## HOME_PORTAL_CURRENT_STATE_20260630\n- Active Homepage URL: https://home.gram1.ru.\n- Homepage container is running and portal HTTP check returned 200 after latest changes.\n- NetBird card points to https://nb.pvepro.ru.\n- Mail card points to https://mail.pvepro.ru.\n- Webmail card was removed; no separate Webmail card is currently configured.\n- Router bookmark points to http://[PRIVATE_IP]:5080 for operator client LAN [PRIVATE_IP].\n- Homepage services.yaml was updated via YAML-aware edit for siteMonitor fields.\n- Current active services.yaml has SITEMONITOR_COUNT=43.\n- Latest validation before this reference update: YAML_ERRORS_LAST_5M=0, HOME_HTTP_CODE=200, HOMEPAGE_RUNNING=true.\n- Do not use sed-only insertion for siteMonitor; use YAML-aware editing and validate YAML before restart.\n- Related active-state checks: NB_LINK_ACTIVE=yes, MAIL_LINK_ACTIVE=yes, WEBMAIL_ACTIVE=no, ROUTER_192_168_1_1_ACTIVE=yes.\n\n## HOME_PORTAL_DIRECT_LINK_FIX_20260630\n- Direct Homepage link correction requested by operator.\n- NetBird: https://nb.pvepro.ru.\n- Mail: https://mail.pvepro.ru.\n- Webmail card removed.\n- Router restored to http://[PRIVATE_IP]:5080.\n\n## HOME_PORTAL_YAML_AWARE_SITEMONITOR_20260630\n- siteMonitor fields were added using YAML-aware PyYAML editing, not line-based sed insertion.\n- Added siteMonitor to 43 service cards.\n- Excluded cards: Homepage, NPMplus, Router and Public Domain.\n- YAML validation passed before restart: YAML_VALIDATE_RC=0.\n- Homepage restarted successfully and reported YAML_ERRORS=0.\n\n## HOMELAB_CLUSTER_BACKLOG_20260630\n- Cluster backlog was created at /etc/pve/HOMELAB_CLUSTER_BACKLOG.md.\n- PBS is intentionally out of scope; backup strategy remains Mail-cloud based.\n- First next action: finish Homepage final validation.\n\n## HOMEPAGE_BACKUP_GAP_CLASSIFICATION_20260630\n- Homepage backup coverage gap classification file created at /etc/pve/HOMEPAGE_BACKUP_GAP_CLASSIFICATION.md.\n- It defines closure requirements for the 13 remaining Homepage backup coverage gap/special-review rows.\n- PBS is intentionally out of scope; closure must use Mail-cloud backups or explicit stateless/config-only classification.\n\n## HOMEPAGE_EXISTING_BACKUP_EVIDENCE_CLOSURE_20260630\n- Homepage backup coverage matrix started closing existing-evidence rows.\n- Router maps to existing Mail-cloud startup/running-config backup proofs when ROUTER_EXISTING_EVIDENCE=yes.\n- MinIO Console maps to existing P2 small-stacks backup/restore proof when MINIO_EXISTING_EVIDENCE=yes.\n- Proof: 364_HOMEPAGE_EXISTING_BACKUP_EVIDENCE_CLOSURE_PROOF.txt.\n\n## HOME_EXTERNAL_CLOUDFLARE_ALEXHOST_20260630\n- Homepage External group includes Cloudflare card: https://dash.cloudflare.com.\n- Homepage External group includes AlexHost Billing card: https://bill.alexhost.com/.\n- Both cards have siteMonitor enabled for green status dots.\n- YAML validation passed, Homepage returned HTTP 200, container running, and no Homepage YAML errors were seen after restart.\n- Proof: 368_HOME_EXTERNAL_CLOUDFLARE_ALEXHOST_PROOF.txt.\n\n## HOME_EXTERNAL_RELAY_REMOVED_20260630\n- Homepage External card/link for relay.pvepro.ru was removed because relay.pvepro.ru had DNS but no reachable HTTP/HTTPS endpoint from edge-vm.\n- Proof: 382_HOME_EXTERNAL_RELAY_REMOVE_PROOF.txt.\n\n## ROUTER_CLI_PERMISSION_LIMIT_20260630\n- Netcraze routerbackup SSH access is read-only for backup: show running-config works, but ACL/config commands are denied.\n- Homepage Moscow Router red-dot fix requires router ACL/Web UI/admin-level change, not routerbackup.\n- Relevant proofs: 391, 393, 394, 395.\n\n## ROUTER_CLI_PROOF_REPAIR_20260630\n- Error register updated for blank proof summary extraction in 395.\n- Corrected derived counts are recorded in 396_ROUTER_CLI_ERROR_REGISTER_REPAIR_PROOF.txt.\n\n## HOME_EXTERNAL_NETBIRD_ALEXHOST_NORMALIZE_20260630\n- Homepage External NetBird and AlexHost Billing href/siteMonitor URLs normalized with trailing slash.\n- Homepage container node checks returned HTTP 200 for both URLs.\n- Proof: 397_HOME_EXTERNAL_NETBIRD_ALEXHOST_NORMALIZE_PROOF.txt.\n\n## ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701\n- Moscow Router ACL is restored and correct after manual Web UI edits.\n- Remaining blocker for Homepage green dot is HTTP/HTTPS 403 from router web service to edge-vm/Bridge1, not network reachability.\n- Relevant proofs: 399, 400, 401, 402.\n\n## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_20260701\n- Moscow Router Homepage card href: http://[PRIVATE_IP]:5080.\n- Moscow Router Homepage siteMonitor: http://[PRIVATE_IP]:18083/router-moscow.\n- Health endpoint service on edge-vm: homelab-router-moscow-health-http.service.\n- Endpoint semantics: returns 200 when TCP to router [PRIVATE_IP]:5080 succeeds, else 503.\n- Relevant proofs: 405, 406, 407.\n\n## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_FIXED_20260701\n- Moscow Router Homepage card target after fixed apply: href http://[PRIVATE_IP]:5080, siteMonitor http://[PRIVATE_IP]:18083/router-moscow.\n- Fixed updater handles Homepage service-name-key YAML shape and property-style YAML shape.\n- Relevant proofs: 405, 406, 407, 408.\n\n## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_APPLIED_EXACT_20260701\n- Moscow Router Homepage card exact active YAML shape was service-key style.\n- Applied href: http://[PRIVATE_IP]:5080.\n- Applied siteMonitor: http://[PRIVATE_IP]:18083/router-moscow.\n- Health endpoint service: homelab-router-moscow-health-http.service on edge-vm.\n- Relevant proofs: 406, 409, 410.\n\n## FORUM_PROD_VM160_REBUILD_BASELINE_20260701\n- Old test VM160 was stopped and destroyed cleanly: proof 419, CONFIG_EXISTS=no, VM160_DISKS_LEFT=0.\n- New VM160 forum-prod was recreated on pve02 with 4 vCPU, 8 GiB RAM, 200G disk, MAC BC:24:11:B6:45:ED, IP [PRIVATE_IP]/24, gateway [PRIVATE_IP].\n- New VM identity confirmed: forum-prod / forum-prod.gram1.ru, Debian 12 bookworm, SSH OK, qemu-agent OK, chrony OK.\n- Swapfile 8G enabled with /swapfile mode 600 and fstab entry: proof 430. Proof 429 is invalid due nested command substitution quoting.\n- SURY PHP repo added; PHP 8.5.7 available and installed with Nginx 1.22.1, MariaDB 10.11.14 and required XenForo modules.\n- Beget-compatible profile applied: memory_limit 256M, post/upload 1024M, max_input_vars 10000, MariaDB utf8mb4/utf8mb4_unicode_ci, innodb_buffer_pool_size 2G.\n- Operator reference copies placed in /root/homelab-operator inside VM160 for local VM command preflight.\n\n## FORUM_PROD_CODEVIPE_FIRST_RECIPE_STRATEGY_20260701\n- Current VM160 is now laboratory state, not final production closure.\n- Bulk import of five forums reached DB/files/nginx/php-fpm ready state, but frontend body stayed empty under PHP 8.5.7.\n- Strategy changed: prove CodeVipe first on controlled runtime, likely PHP 8.3 or PHP 8.2, then destroy VM160 and rebuild all five forums from scratch using the proven recipe.\n- Do not publish Cloudflare/NPMplus routes for these forums until CodeVipe recipe is proven and final rebuild is complete.\n- Proofs: failed/diagnostic chain 451-480; next snapshot proof 481 before further lab work.\n\n## FORUM_PROD_FRESH5_DEPLOYED_OK_20260701\n- VM160 forum-prod on pve02, IP [PRIVATE_IP].\n- Runtime: Debian 12, Nginx, MariaDB 10.11, PHP 8.3-FPM.\n- Five XenForo forums deployed from fresh ZIP backups: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- Webroots: /var/www/forums/{codevipe,gamevipe,hkmods,zakrutim,dsmods}/public.\n- DBs: xf_codevipe, xf_gamevipe, xf_hkmods, xf_zakrutim, xf_dsmods.\n- Final health proof inside VM: /root/evidence/513_FORUM_PROD_FRESH5_FINAL_HEALTH_PROOF.txt.\n- Final result: all five frontend/admin HTTP 200, www redirects 301, internal_data 403, no new XenForo errors.\n- Lesson: fresh file ZIPs contain populated internal_data/code_cache; broken branch lacked compiled template cache.\n\n## OPERATOR_RULE_CONTINUE_WITH_NEXT_COMMAND_20260701\n- Rule: when an infrastructure step ends at an expected prompt or safe checkpoint, do not stop with only a status summary.\n- Rule: immediately provide the next executable command in the same response.\n- Applies to: SSH enter/exit points, VM prompt confirmations, snapshot confirmations, file copy confirmations, successful health checks, and other obvious next-step transitions.\n- Exception: stop only if the previous output contains an error, ambiguity, dangerous state, missing file, secret exposure risk, or a required user decision.\n\n## OPERATOR_RULE_BATCH_2_3_COMMANDS_20260701\n- Rule: when safe, provide 2-3 sequential executable commands in one response instead of stopping after each expected checkpoint.\n- Applies after: successful prompt transitions, clean health checks, completed copies, completed snapshots, and non-destructive inventory.\n- Do not batch commands across: destructive operations, public exposure, DNS/SSL changes, secret handling, unclear errors, missing files, or required user decisions.\n- Each command must still print ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK before work.\n\n## FORUM_PROD_FRESH5_PUBLIC_CUTOVER_OK_20260701\n- Public cutover completed for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- Cloudflare A records for main and www names point to edge public IP 95.84.154.183.\n- Edge NPMplus manual routes terminate TLS with real Let’s Encrypt certificates and proxy to forum-prod VM160 at [PRIVATE_IP]:80.\n- Final public proof: /root/evidence/546_PUBLIC_FORUMS_FINAL_DNS_HTTPS_PROOF.txt on pve01.\n- Expected public state: main domains HTTPS 200 with ssl_verify_result 0; HTTP redirects to HTTPS; www redirects to main domain through forum-prod canonical vhosts.\n\n## EDGE_FORUM_PUBLICATION_BACKUP_20260701\n- Edge NPMplus forum publication backup created after public cutover.\n- Backup location: /opt/npmplus/manual-backups/forum-public-ok-*.tar.gz on edge-vm [PRIVATE_IP].\n- Includes manual proxy_host configs 200-204 and Let’s Encrypt forum certificates under /opt/npmplus/tls/forum-certs.\n\n## FORUM_CERT_RENEWAL_CONFIGURED_20260701\n- Edge certificate renewal configured on edge-vm [PRIVATE_IP].\n- Token stored at /opt/npmplus/secure/forum_cf_token.env mode 600 root-only.\n- Renewal script: /opt/npmplus/scripts/renew-forum-certs.sh.\n- Cron: /etc/cron.d/forum-cert-renew, daily 03:17.\n- Proof: /root/evidence/550_EDGE_FORUM_CERT_RENEWAL_SETUP_PROOF.txt on pve01.\n\n## CLOUDFLARE_FORUM_DNS_AUDIT_OK_20260701\n- Full Cloudflare DNS audit completed for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- Proof: /root/evidence/554_CLOUDFLARE_FULL_DNS_AUDIT_PROOF.txt on pve01.\n- Result: root and www A records for all five zones point to 95.84.154.183.\n- Result: no root/www AAAA records, no root/www CNAME conflicts, no records with old IP 87.236.18.45.\n- Remaining cdn.* CNAME and TXT/DMARC/DKIM records were observed and not changed.\n\n## FORUMS_FINAL_DNS_AUDIT_SNAPSHOT_OK_20260701\n- Final post-DNS-audit snapshot completed after shortening Proxmox snapshot name.\n- Snapshot name: forum-dns-ok-065203Z\n- Proof: /root/evidence/557_FINAL_SHORT_SNAPSHOT_AFTER_DNS_AUDIT_PROOF.txt on pve01.\n- Current public state: all five main domains resolve to 95.84.154.183 and return HTTPS 200 with valid certificates.\n\n## FORUM_LOCAL_BACKUP_CONFIGURED_20260701\n- Local forum backup configured inside VM160 forum-prod.\n- Script: /root/scripts/forum-backup.sh.\n- Cron: /etc/cron.d/forum-local-backup, daily 02:42.\n- Retention: removes per-run directories older than 14 days; compressed archives remain under /var/backups/forums.\n- Proof: /root/evidence/560_FORUM_BACKUP_SCRIPT_SETUP_PROOF.txt on pve01.\n\n## FORUM_MAIL_NOREPLY_AND_SPF_CLEANUP_20260701\n- XenForo email fields defaultEmailAddress/contactEmailAddress/bounceEmailAddress/emailSenderName set to noreply@pvepro.ru for all five forums.\n- SPF TXT on codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru changed from stale 87.236.18.45 to v=spf1 -all.\n- DKIM, DMARC and cdn.* CNAME records were not changed.\n- Proofs: 564_UPDATE_XENFORO_EMAIL_NOREPLY_PROOF.txt, 565_CF_UPDATE_FORUM_SPF_NOREPLY_PROOF.txt, 566_FORUM_MAIL_NOREPLY_FINAL_AUDIT_PROOF.txt.\n\n## FORUM_CDN_RECORDS_DELETED_20260701\n- Deleted cdn.* CNAME records for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- Reason: old cdn.* records pointed to previous provider/trbcdn and no current public/local forum references were observed.\n- Root and www A records remain on 95.84.154.183.\n- Proofs: /root/evidence/567_CF_DELETE_FORUM_CDN_RECORDS_PROOF.txt and /root/evidence/568_CDN_DELETE_FINAL_AUDIT_REFERENCE_PROOF.txt on pve01.\n\n## FORUM_SPF_DEDUPED_AND_CDN_DELETED_20260701\n- Fixed duplicate SPF state caused by earlier failed cleanup attempt.\n- Each of codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru now has exactly one SPF TXT: v=spf1 -all.\n- Old SPF references to 87.236.18.* are absent.\n- cdn.* CNAME records are absent.\n- Public HTTPS remained healthy for all five forums.\n- Proofs: 569_CF_FIX_DUPLICATE_SPF_RECORDS_PROOF.txt and 570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt.\n\n## FORUM_PUBLIC_DNS_MAIL_CLEAN_FINAL_20260701\n- Public web and mail-related DNS cleanup completed for five forum domains.\n- @ and www A records point to 95.84.154.183.\n- cdn.* CNAME records removed.\n- SPF deduped to exactly one TXT: v=spf1 -all on each forum domain.\n- Old provider IP 87.236.18.* absent from forum domain TXT records.\n- XenForo visible email identity changed to noreply@pvepro.ru on all forums.\n- Proofs: 570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt, 571_FORUM_MAIL_TRANSPORT_READINESS_PROOF.txt, 572_FORUM_PUBLIC_AND_MAIL_DNS_FINAL_SUMMARY_PROOF.txt.\n\n## FORUM_SMTP_TRANSPORT_PAUSED_20260701\n- Forum XenForo email identity is noreply@pvepro.ru, but actual SMTP transport is paused.\n- Reason: failed SMTP authentication triggered Mailcow netfilter ban for 95.84.154.183.\n- Broken msmtp secret/config files were removed from forum-prod.\n- Do not re-enable SMTP until the mailbox/app password is rotated and tested with a single controlled attempt.\n- Proof: /root/evidence/590_FORUM_SMTP_FAILED_CONFIG_DISABLED_PROOF.txt.\n\n## POST_INCIDENT_STABLE_STATE_20260701\n- Incident after failed forum SMTP testing resolved.\n- Mailcow and NetBird are reachable again: mail.pvepro.ru/admin and nb.pvepro.ru return HTTP 200.\n- Mailcow netfilter ban for 95.84.154.183 was removed on VPS mail.pvepro.ru.\n- Forum sites remain healthy over HTTPS.\n- Forum SMTP transport remains paused; broken msmtp configs/secrets removed from forum-prod.\n- Do not retry SMTP until the mailbox/app password is rotated.\n- Proofs: 589_POST_UNBAN_MAILCOW_NETBIRD_VERIFY.txt on VPS, 590_FORUM_SMTP_FAILED_CONFIG_DISABLED_PROOF.txt, 591_POST_INCIDENT_PUBLIC_STATUS_PROOF.txt, 592_POST_INCIDENT_FINAL_STABLE_STATE_PROOF.txt.\n\n## FORUM_SMTP_ONE_SHOT_TEST_OK_20260701\n- One controlled SMTP test from forum-prod to mail.pvepro.ru:587 succeeded using rotated credentials.\n- No persistent forum SMTP config was enabled.\n- One-shot secret/config files were removed after the test.\n- Mailcow and NetBird remained reachable after the test.\n- Proofs: /root/evidence/594_FORUM_SMTP_ONE_SHOT_TEST_PROOF.txt and /root/evidence/595_SMTP_TEST_AFTERCHECK_PROOF.txt.\n\n## SMTP_ONESHOT_OK_STABLE_FINAL_20260701\n- Rotated SMTP credential was tested once from forum-prod to mail.pvepro.ru:587 successfully.\n- No persistent SMTP config or secret was left on forum-prod after the one-shot test.\n- Mailcow and NetBird remained reachable after the test.\n- Forum websites remained healthy.\n- Persistent forum SMTP is still not enabled; enable it only with an explicit command and hidden password input.\n- Proofs: 594_FORUM_SMTP_ONE_SHOT_TEST_PROOF.txt, 595_SMTP_TEST_AFTERCHECK_PROOF.txt, 596_SMTP_ONESHOT_OK_FINAL_STABLE_PROOF.txt.\n\n## FORUM_PERSISTENT_MSMTP_ENABLED_20260701\n- Persistent msmtp-mta transport enabled on forum-prod as /usr/sbin/sendmail.\n- SMTP relay: mail.pvepro.ru:587 STARTTLS, sender noreply@pvepro.ru.\n- Secret stored only as root:www-data 0640 base64 file under /etc/msmtp/pvepro.env; no raw SMTP_PASS file remains.\n- PHP mail() as www-data succeeded after persistent config installation.\n- Mailcow and NetBird remained reachable after enabling persistent forum SMTP.\n- Proofs: /root/evidence/597_ENABLE_FORUM_PERSISTENT_MSMTP_PROOF.txt and /root/evidence/598_PERSISTENT_MSMTP_FINAL_VERIFY_PROOF.txt.\n\n## FORUM_PERSISTENT_SMTP_FINAL_STABLE_20260701\n- Persistent forum SMTP via msmtp is enabled on forum-prod.\n- PHP mail() as www-data succeeded after installation.\n- Forum domains remain healthy over HTTPS.\n- Mailcow and NetBird remain reachable after enabling persistent forum SMTP.\n- Secret storage is root:www-data 0640 under /etc/msmtp/pvepro.env; raw [SENSITIVE_PATH] SMTP_PASS file is absent.\n- Snapshot after enable: forum-smtp-on-080840Z.\n- Proofs: 597_ENABLE_FORUM_PERSISTENT_MSMTP_PROOF.txt, 598_PERSISTENT_MSMTP_FINAL_VERIFY_PROOF.txt, 600_FORUM_PERSISTENT_SMTP_FINAL_STABLE_PROOF.txt.\n\n## FORUM_PROD_FRESH5_MAILRU_CLOUD_BACKUP_OK_20260701\n- Existing Mail.ru Cloud rclone crypt remotes on pve02 were reused; no new cloud account was configured.\n- Old codevipe-cloud-backup.timer was stale after fresh five-forum rebuild and is disabled.\n- New pve02 timer forum-prod-fresh5-cloud-backup.timer uploads current forum-prod local archives from VM160 /var/backups/forums.\n- Current backup contains all five forums: codevipe, gamevipe, hkmods, zakrutim, dsmods.\n- Upload target: pve02-mail-01/02/03/04-crypt under app/forums/forum-prod/fresh5/.\n- Archive is split into 1500M parts before upload, with SHA256SUMS and MANIFEST.txt.\n- First fresh5 cloud upload proof: /root/evidence/608_FORUM_PROD_FRESH5_CLOUD_BACKUP_FIRST_RUN_PROOF.txt.\n\n## FORUM_PROD_FRESH5_MAILRU_CLOUD_RESTORE_DRILL_OK_20260701\n- Restore drill from Mail.ru Cloud crypt remote completed for forum-prod fresh5 backup.\n- Remote source used: pve02-mail-01-crypt app/forums/forum-prod/fresh5 latest timestamp.\n- Downloaded split parts, MANIFEST.txt, SHA256SUMS and SHA256SUMS.local were verified.\n- Part SHA256 verification passed.\n- Split archive was reconstructed and full archive hash matched SHA256SUMS.local.\n- tar listing confirmed DB dumps and file archives for codevipe, gamevipe, hkmods, zakrutim and dsmods.\n- Proof: /root/evidence/612_FORUM_FRESH5_CLOUD_RESTORE_DRILL_CORRECTED_PROOF.txt.\n\n## FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_OK_20260701\n- Five public XenForo forums are healthy over HTTPS.\n- Mailcow admin and NetBird UI are reachable after persistent forum SMTP enablement.\n- Persistent forum SMTP via msmtp is enabled and tested.\n- Local backup exists on VM160 under /var/backups/forums.\n- Mail.ru Cloud fresh5 backup is enabled on pve02 via forum-prod-fresh5-cloud-backup.timer.\n- Old CodeVipe-only cloud timer is disabled.\n- Cloud restore drill passed: split parts verified, archive reconstructed, SHA256 matched, tar contained DB/file archives for all five forums.\n- Final rollup proof: /root/evidence/614_FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_PROOF.txt.\n\n## FORUM_XENFORO_JOB_RUNNER_CONFIGURED_20260701\n- XenForo CLI job runner configured inside VM160 forum-prod.\n- Script: /root/scripts/forum-xenforo-run-jobs.sh.\n- Cron: /etc/cron.d/forum-xenforo-run-jobs every 5 minutes.\n- Purpose: process XenForo job queues and cron tasks independent of forum traffic.\n- First run proof: /root/evidence/617_XENFORO_RUN_JOBS_ONCE_PROOF.txt.\n- Cron verification proof: /root/evidence/618_XENFORO_JOB_RUNNER_CRON_CONFIG_PROOF.txt.\n\n## FORUM_XENFORO_CRON_DAEMON_AND_JOBRUNNER_OK_20260701\n- cron daemon inside VM160 forum-prod is active/enabled and was proven by temporary /etc/cron.d execution proof.\n- XenForo job runner script executed successfully after cron daemon verification.\n- Due XenForo cron count returned to zero after run.\n- Proof: /root/evidence/621_FORUM_CRON_DAEMON_JOBRUNNER_VERIFY_PROOF.txt.\n\n## FORUM_DBTECH_TOR_TIMEOUT_CLASSIFIED_20260701\n- Each forum had one historical XenForo error_log row for cURL timeout to check.torproject.org/torbulkexitlist.\n- Source addon: DBTech/Security DragonByte Security 5.0.0.\n- dbtech_security_tornodes option is 0 on all five forums.\n- Classified as non-blocking historical external network timeout/noise; not a forum runtime failure.\n- Error rows were not deleted.\n- Proof: /root/evidence/622_TOR_TIMEOUT_CLASSIFICATION_FINAL_PROOF.txt.\n\n## FORUM_XENFORO_BUILTIN_MAIL_DELIVERED_BUT_SPAM_20260701\n- Built-in XenForo outgoing email test from site \"Моды для Hollow Knight\" was delivered to aleisaev@yandex.ru.\n- Sender was noreply@pvepro.ru.\n- Yandex placed the message in Spam with warning that links/images were disabled.\n- This proves forum SMTP transport works, but deliverability/reputation needs tuning.\n- Custom proof 623 is invalid due /root path permission issue and is superseded by this user-observed built-in test.\n- Next evidence needed: Yandex raw message headers, especially Authentication-Results and DKIM-Signature.\n- Proof: /root/evidence/627_XENFORO_BUILTIN_MAIL_DELIVERED_SPAM_CLASSIFICATION_PROOF.txt.\n\n## FORUM_YANDEX_DELIVERABILITY_SPAM_WITH_SPF_DKIM_PASS_20260701\n- Built-in XenForo outgoing email test was delivered to aleisaev@yandex.ru but placed in Spam.\n- User-provided Yandex headers showed SPF pass and DKIM pass for pvepro.ru.\n- Return-Path, From and DKIM domain aligned on pvepro.ru.\n- Yandex spam score observed: X-Yandex-Spam: 4.\n- Classification: SMTP/Mailcow/XenForo transport is OK; remaining issue is deliverability/reputation/content filtering.\n- DNS change is not required based on this header proof.\n- Next actions: mark as Not spam in Yandex, add/check pvepro.ru in Yandex Postmaster, warm up sender reputation, optionally plan per-forum sender domains later.\n\n## FORUM_PROJECT_FINAL_ACCEPTANCE_OK_20260701\n- Five XenForo forums are published and healthy over HTTPS: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- VM160 forum-prod is the active production forum VM on pve02 at [PRIVATE_IP].\n- Runtime services are configured: nginx, MariaDB, PHP 8.3 FPM, qemu-agent, cron, msmtp/sendmail transport.\n- NPMplus on edge terminates TLS and proxies all five forums to VM160.\n- Let’s Encrypt certificates are active for all five domains.\n- Forum SMTP transport works through noreply@pvepro.ru; Yandex headers proved SPF pass and DKIM pass. Spam placement is classified as reputation/content deliverability, not server failure.\n- XenForo job runner cron is configured and cron daemon execution was proven.\n- Local backup is configured on VM160 and Mail.ru Cloud fresh5/all-forums backup is configured on pve02.\n- Cloud restore drill passed: split parts verified, archive SHA matched, tar contained DB/file/config archives for all five forums.\n- DBTech Tor timeout rows are classified as historical external timeout noise and were not deleted.\n- Final proof: /root/evidence/629_FORUM_PROJECT_FINAL_ACCEPTANCE_PROOF.txt.\n\n## FORUM_NEXT_CHAT_HANDOFF_FULL_STATE_20260701\n\n### Start point\n- Start shell: root@pve01.\n- Canonical truth files:\n - /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\n - /etc/pve/31_HOMELAB_REFERENCE.md\n- Before any infra command, strictly check both files and relevant context blocks.\n- Every infra command must print:\n - ERROR_REGISTER_CHECK=OK\n - REFERENCE_CHECK=OK\n\n### Production forum VM\n- VMID: 160.\n- Name: forum-prod.\n- Node: pve02.\n- IP: [PRIVATE_IP].\n- OS: Debian 12.\n- Runtime: nginx, MariaDB 10.11, PHP 8.3 FPM, qemu-agent, cron, msmtp/sendmail transport.\n- Final accepted snapshot: forum-final-accepted-091934Z.\n- Mail final snapshot: forum-mail-ok-091815Z.\n- Postlaunch snapshot: forum-postlaunch-ok-085501Z.\n\n### Public forums\n- codevipe.ru -> Форум CodeVipe.\n- gamevipe.ru -> Форум GameVipe.\n- hkmods.ru -> Моды для Hollow Knight.\n- zakrutim.ru -> Консервирование и закрутки.\n- dsmods.ru -> Секреты и моды Doom.\n- All five are public HTTPS 200 with valid TLS.\n- Root/www DNS points through edge public IP 95.84.154.183.\n- NPMplus on edge terminates TLS and proxies to http://[PRIVATE_IP]:80.\n\n### Mail\n- Forum sender: noreply@pvepro.ru.\n- Transport: XenForo sendmail -> msmtp -> mail.pvepro.ru.\n- Built-in XenForo mail test reached Yandex.\n- Yandex headers showed SPF pass and DKIM pass for pvepro.ru.\n- Yandex placed the test in Spam with X-Yandex-Spam: 4.\n- Classification: transport/authentication OK; remaining issue is sender reputation/content filtering.\n- DNS change is not required from the captured header proof.\n- Manual next action: click “Не спам!” in Yandex and add/check pvepro.ru in Yandex Postmaster.\n\n### Backups and restore\n- Local backup configured inside VM160.\n- Local archive path pattern: /var/backups/forums/forum-backup-*.tar.gz.\n- pve02 Mail.ru Cloud fresh5/all-forums backup configured.\n- New timer: forum-prod-fresh5-cloud-backup.timer active/enabled.\n- Old codevipe-only timer disabled/inactive.\n- Cloud remotes:\n - pve02-mail-01-crypt\n - pve02-mail-02-crypt\n - pve02-mail-03-crypt\n - pve02-mail-04-crypt\n- Latest verified cloud stage during final acceptance: 20260701T083354Z.\n- Restore drill passed: split parts verified, archive SHA matched, tar contained DB/file/config archives for all five forums.\n\n### XenForo jobs and cron\n- cron daemon was missing, then installed and enabled.\n- /etc/cron.d execution was proven by temporary cron proof.\n- XenForo job runner configured:\n - /root/scripts/forum-xenforo-run-jobs.sh\n - /etc/cron.d/forum-xenforo-run-jobs\n- Purpose: process XenForo jobs and cron tasks independent of visitor traffic.\n\n### DBTech/Tor timeout\n- Each forum had one historical xf_error_log row for cURL timeout to check.torproject.org.\n- Source addon: DBTech/Security DragonByte Security 5.0.0.\n- dbtech_security_tornodes=0 on all five forums.\n- Classified as non-blocking historical external timeout noise.\n- Rows were not deleted.\n\n### Key final proofs\n- /root/evidence/629_FORUM_PROJECT_FINAL_ACCEPTANCE_PROOF.txt\n- /root/evidence/628_FORUM_MAIL_DELIVERABILITY_FINAL_SNAPSHOT_PROOF.txt\n- /root/evidence/627_YANDEX_HEADER_DELIVERABILITY_CLASSIFICATION_PROOF.txt\n- /root/evidence/622_TOR_TIMEOUT_CLASSIFICATION_FINAL_PROOF.txt\n- /root/evidence/621_FORUM_CRON_DAEMON_JOBRUNNER_VERIFY_PROOF.txt\n- /root/evidence/614_FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_PROOF.txt\n- /root/evidence/613_FORUM_FRESH5_CLOUD_RESTORE_DRILL_FINALIZE_PROOF.txt\n- /root/evidence/612_FORUM_FRESH5_CLOUD_RESTORE_DRILL_CORRECTED_PROOF.txt\n- /root/evidence/609_FORUM_PROD_FRESH5_CLOUD_BACKUP_FINALIZE_PROOF.txt\n- /root/evidence/608_FORUM_PROD_FRESH5_CLOUD_BACKUP_FIRST_RUN_PROOF.txt\n- /root/evidence/600_FORUM_PERSISTENT_SMTP_FINAL_STABLE_PROOF.txt\n- /root/evidence/570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt\n- /root/evidence/560_FORUM_BACKUP_SCRIPT_SETUP_PROOF.txt\n\n## PARKED_DOMAINS_PLACEHOLDER_PUBLIC_OK_20260701\n\n- Parked/placeholder public page is live for three unused domains: newfi.ru, hapusya.ru and kingofwolk.ru.\n- Public DNS root and www hostnames already point to edge public IP 95.84.154.183.\n- Local placeholder service on edge-vm: homelab-parked-domains-http.service, enabled/active, serves marker PARKED_PAGE_OK on http://127.0.0.1:18088.\n- NPMplus manual managed route files: /data/nginx/proxy_host/998.conf for newfi.ru, 997.conf for hapusya.ru and 996.conf for kingofwolk.ru.\n- Public HTTPS validation from pve01 after final route-only finalize returned HTTP 200 and PARKED_PAGE_OK for all six hostnames: root and www for all three domains.\n- Certificates were issued by certbot DNS-01 using Cloudflare API hooks; HTTP-01 must not be retried for these parked domains without a new plan because earlier attempts hit NPMplus default redirect/include-order behavior.\n- Cloudflare token value is not stored in the reference; token file path only: /opt/npmplus/secure/parked_cf_token, root-owned mode 600.\n- Certificate names on edge-vm: parked-newfi.ru, parked-hapusya.ru and parked-kingofwolk.ru; observed expiry during setup: 2026-09-29.\n- Certbot deploy hook installed: /etc/letsencrypt/renewal-hooks/deploy/parked-domains-npmplus-deploy.sh.\n- Health file: /var/lib/homelab-health/parked-domains-public.txt.\n- Final route backup: /opt/npmplus/manual-backups/parked-stage16-route-only-20260701T155843Z.\n- Final proof: /root/evidence/645_PARKED_DOMAINS_STAGE16_ROUTE_ONLY_FINALIZE_PROOF.txt.\n\n## GRAM1_ROOT_WWW_PLACEHOLDER_PUBLIC_OK_20260701\n\n- gram1.ru root and www.gram1.ru are routed to the existing parked placeholder page on edge-vm.\n- Existing gram1.ru subdomain routes were not changed.\n- NPMplus managed route file: /data/nginx/proxy_host/995.conf.\n- Upstream placeholder: http://127.0.0.1:18088.\n- Certificate name on edge-vm: parked-gram1.ru.\n- Certificate issuance mode: certbot DNS-01 with dedicated gram1 Cloudflare token file /opt/npmplus/secure/gram1_cf_token.\n- Token value is not stored in reference; token file is root-owned mode 600.\n- Certbot deploy hook: /etc/letsencrypt/renewal-hooks/deploy/gram1-root-npmplus-deploy.sh.\n- Health file: /var/lib/homelab-health/gram1-root-placeholder.txt.\n- Closure proof: /root/evidence/653_GRAM1_ROOT_WWW_PLACEHOLDER_CLOSE_STAGE19_PROOF.txt.\n\n## PVEPRO_ROOT_WWW_EDGE_LANDING_PUBLIC_OK_20260701\n\n- pvepro.ru and www.pvepro.ru root/www were moved from the Mailcow VPS default web surface to an edge landing page.\n- mail.pvepro.ru and nb.pvepro.ru were intentionally not changed.\n- Cloudflare A records for pvepro.ru and www.pvepro.ru point to edge public IP 95.84.154.183.\n- NPMplus managed route file: /data/nginx/proxy_host/994.conf.\n- Landing service on edge-vm: homelab-pvepro-landing-http.service on http://127.0.0.1:18089.\n- Landing marker: PVEPRO_LANDING_OK.\n- Certificate name on edge-vm: landing-pvepro.ru.\n- Certificate issuance mode: certbot DNS-01 with dedicated pvepro Cloudflare token file /opt/npmplus/secure/pvepro_cf_token.\n- Token value is not stored in reference; token file is root-owned mode 600.\n- Certbot deploy hook: /etc/letsencrypt/renewal-hooks/deploy/pvepro-root-npmplus-deploy.sh.\n- Health file: /var/lib/homelab-health/pvepro-root-landing.txt.\n- Closure proof: /root/evidence/658_PVEPRO_STAGE23_FINAL_READONLY_CLOSE_PROOF.txt.\n\n## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701\n\n- taftauto.ru is the dacha router domain.\n- Current public A record observed during audit: 194.33.48.131.\n- Router model family: Netcraze-like, same as Moscow router family by operator statement.\n- Safe private management access is not available yet.\n- Do not expose the router admin interface publicly for certificate automation.\n- Certificate auto-renewal/deploy to the router is intentionally blocked until a private access path exists.\n- Recommended future path: establish VPN/NetBird/WireGuard or reverse-proxy/private management endpoint first; then issue DNS-01 certificate on a trusted node and deploy cert/key to the router only over that private path.\n- Closure status: documented blocker, not runtime outage.\n- Read-only audit proof: /root/evidence/654_PVEPRO_TAFTAUTO_EXTERNAL_READONLY_AUDIT_PROOF.txt.\n\n## DOMAIN_PORTFOLIO_FINAL_STATUS_20260701\n\n- Public parked placeholder domains closed: newfi.ru, hapusya.ru, kingofwolk.ru.\n- gram1.ru root and www.gram1.ru closed on the same placeholder page; existing gram1.ru service subdomains were not changed.\n- pvepro.ru root and www.pvepro.ru closed on a separate edge landing page; mail.pvepro.ru and nb.pvepro.ru remain on the external VPS/IPs and were validated after the change.\n- Forum domains remain XenForo on forum-prod through edge NPMplus: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru.\n- taftauto.ru is documented as blocked for certificate autodeploy until private router management access exists.\n- Forum renewal one-shot proof from Stage18 returned OK.\n- Parked certbot dry-run passed for newfi.ru and hapusya.ru; kingofwolk.ru dry-run hit transient Let’s Encrypt service-busy/rateLimited after a valid active certificate and working public HTTPS were already confirmed.\n- Current finalization proof: /root/evidence/661_DOMAIN_PORTFOLIO_FINAL_CLOSE_AND_TAFTAUTO_BLOCKED_PROOF.txt.\n\n## TAFTAUTO_WG_PRIVATE_MANAGEMENT_OK_20260702\n- Public SSH 194.33.48.131:22 closed.\n- WireGuard management path OK: edge [PRIVATE_IP] -> dacha router [PRIVATE_IP].\n- Router HTTP over WG returns HTTP/1.1 200 OK / Ndm-Sysmode: router.\n- Proof: /root/evidence/665_TAFTAUTO_WG_PRIVATE_MANAGEMENT_OK_20260702_PROOF.txt\n\n## TAFTAUTO_WG_PSK_ROTATED_AND_MANAGEMENT_OK_20260702\n- WireGuard PSK rotated after leaked self-test.\n- Dacha active interface after reimport: Wireguard1.\n- Public SSH remains closed.\n- Private management OK: edge [PRIVATE_IP] -> dacha [PRIVATE_IP] via WG.\n- Proof: /root/evidence/666_TAFTAUTO_WG_PSK_ROTATED_AND_MANAGEMENT_OK_20260702_PROOF.txt\n\n## TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702\n- Supersedes proof 666 because proof 666 showed PUBLIC_SSH_22_STILL_OPEN.\n- WireGuard PSK rotated.\n- Public SSH closed.\n- Private WG management OK: edge [PRIVATE_IP] -> dacha [PRIVATE_IP].\n- Proof: /root/evidence/667_TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702_PROOF.txt\n\n## TAFTAUTO_CERT_DNS01_ISSUED_20260702\n- DNS-01 certificate issued on edge-vm for taftauto.ru and www.taftauto.ru.\n- Cert name: router-taftauto.ru.\n- Cert path: /etc/letsencrypt/live/router-taftauto.ru/fullchain.pem.\n- Key path: /etc/letsencrypt/live/router-taftauto.ru/privkey.pem.\n- Proof: /root/evidence/668_TAFTAUTO_CERT_DNS01_ISSUED_20260702_PROOF.txt\n\n## TAFTAUTO_PUBLIC_EDGE_WG_ROUTER_FINAL_OK_20260702\n- Supersedes failed/partial proof 669 because direct SSH was open during that run.\n- taftauto.ru and www.taftauto.ru point to edge public IP 95.84.154.183.\n- NPMplus route: /opt/npmplus/nginx/proxy_host/993.conf.\n- TLS terminates on edge using certbot DNS-01 cert router-taftauto.ru.\n- Upstream is private WireGuard path: edge [PRIVATE_IP] -> dacha router [PRIVATE_IP]:80.\n- Public HTTPS returns router page with x-taftauto-router: managed and ndm-sysmode: router.\n- Direct dacha public SSH is closed; WG SSH remains open.\n- Proof: /root/evidence/670_TAFTAUTO_PUBLIC_EDGE_WG_ROUTER_FINAL_OK_20260702_PROOF.txt\n\n## HOMEPAGE_EXTERNAL_SMARTAPE_AND_SITES_OK_20260702\n- Homepage services.yaml repaired with YAML-aware edit after bad indentation issue.\n- SmartApe card added to External Homelabs.\n- Router Moscow card removed.\n- Sites category added with 11 site cards: CodeVipe, GameVipe, HKMods, Zakrutim, DSMods, Newfi, Hapusya, KingOfWolk, Gram1, PVEPro, TaftAuto.\n- Proof: /root/evidence/671_HOMEPAGE_EXTERNAL_SMARTAPE_AND_SITES_OK_20260702_PROOF.txt\n\n## TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK_20260702\n- Certbot renew dry-run for router-taftauto.ru succeeds with manual DNS-01 hooks.\n- Deploy hook installed: /etc/letsencrypt/renewal-hooks/deploy/taftauto-router-npmplus-deploy.sh.\n- Deploy hook copies renewed cert/key into NPMplus /data/tls/router-certs/taftauto.ru and reloads nginx after nginx -t.\n- Proof: /root/evidence/672_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK_20260702_PROOF.txt\n\n## TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_INSTALLED_DRYRUN_RATELIMITED_20260702\n- Supersedes partial proof 672.\n- Deploy hook installed and manual deploy invocation works.\n- NPMplus cert copy and nginx config validate OK.\n- Public taftauto.ru remains OK.\n- Certbot dry-run retry is deferred due Let's Encrypt rateLimited / Service busy.\n- Proof: /root/evidence/673_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_INSTALLED_DRYRUN_RATELIMITED_20260702_PROOF.txt\n\n## VPN_GRAM1_ALIAS_ROUTES_REMOVED_20260702\n- Removed obsolete NPMplus route aliases without Homepage cards: auth.vpn.gram1.ru, gitea.vpn.gram1.ru, homepage.vpn.gram1.ru, kuma.vpn.gram1.ru.\n- NPMplus nginx config validates after removal.\n- Proof: /root/evidence/674_VPN_GRAM1_ALIAS_ROUTES_REMOVED_20260702_PROOF.txt\n\n## HOMELAB_HOMEPAGE_AND_TAFTAUTO_FINAL_OK_20260702\n- Homepage final layout/category state confirmed: YAML validates, categories use row/columns layout, logs show no YAMLException.\n- TaftAuto public HTTPS works for taftauto.ru and www.taftauto.ru through edge/NPMplus.\n- Direct public SSH to dacha router is closed.\n- Private WG SSH to dacha router remains open.\n- NPMplus nginx config validates.\n- Proof: /root/evidence/677_HOMELAB_HOMEPAGE_AND_TAFTAUTO_FINAL_OK_20260702_PROOF.txt\n\n## CLOUDFLARE_TOKEN_AUDIT_20260702\n- Cloudflare token files audited without printing secrets.\n- Token files exist, expected permissions were checked, tokens verify active, and expected zone access was checked.\n- Proof: /root/evidence/678_CLOUDFLARE_TOKEN_AUDIT_20260702_PROOF.txt\n\n## EVIDENCE_REVIEW_INDEX_20260702\n- Read-only evidence index created for review/superseded proof cleanup planning.\n- No evidence files were deleted or modified.\n- Proof: /root/evidence/679_EVIDENCE_REVIEW_INDEX_20260702_PROOF.txt\n\n## EVIDENCE_SUPERSEDED_MAP_20260702\n- Evidence superseded map created. No evidence files were deleted.\n- 666, 669, 672_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK, and 676 are not authoritative for final state.\n- 677 is current final Homepage/TaftAuto authority; 678 is current Cloudflare token audit authority.\n- Proof: /root/evidence/680_EVIDENCE_SUPERSEDED_MAP_20260702_PROOF.txt\n\n## HOMEPAGE_LOCAL_HEALTH_MONITORS_OK_20260702\n- Local edge health endpoint is used for Homepage monitoring of Роутер Москва and NPMplus.\n- Роутер Москва href remains external, while siteMonitor uses http://[PRIVATE_IP]:18091/router-moscow.\n- NPMplus siteMonitor uses http://[PRIVATE_IP]:18091/npmplus.\n- Homepage YAML validates and logs show no YAMLException.\n- Proof: /root/evidence/681_HOMEPAGE_LOCAL_HEALTH_MONITORS_OK_20260702_PROOF.txt\n\n## HOMEPAGE_SELF_CARD_REMOVED_FROM_CORE_20260702\n- Removed Homepage self-referential card from Core.\n- Homepage YAML validates and logs show no YAMLException.\n- Proof: /root/evidence/682_HOMEPAGE_SELF_CARD_REMOVED_FROM_CORE_20260702_PROOF.txt\n\n## CERT_RENEW_HOOKS_READONLY_AUDIT_20260702\n- Read-only audit of cert renewal cron entries, renewal configs, deploy hooks, and related script presence completed.\n- No certbot renew/dry-run was executed.\n- Proof: /root/evidence/683_CERT_RENEW_HOOKS_READONLY_AUDIT_20260702_PROOF.txt\n\n## CERT_RENEW_MAPPING_READONLY_AUDIT_20260702\n- Read-only mapping audit completed for renewal confs, deploy hooks, and forum renewal script.\n- No certbot renew/dry-run was executed and no secrets were printed.\n- Proof: /root/evidence/684_CERT_RENEW_MAPPING_READONLY_AUDIT_20260702_PROOF.txt\n\n## CERT_DEPLOY_HOOK_LINEAGE_GUARDS_OK_20260702\n- Deploy hooks hardened with RENEWED_LINEAGE guards for gram1, parked domains, and pvepro.\n- TaftAuto hook already had lineage guard and remains guarded.\n- bash -n validates all checked deploy hooks.\n- No certbot renew/dry-run was executed.\n- Proof: /root/evidence/685_CERT_DEPLOY_HOOK_LINEAGE_GUARDS_OK_20260702_PROOF.txt\n\n## PUBLIC_HTTPS_AND_CERT_EXPIRY_READONLY_20260702\n- Read-only public HTTPS and certificate expiry snapshot completed for forum, parked, gram1, pvepro, and taftauto domains.\n- No certbot renew/dry-run was executed.\n- Proof: /root/evidence/686_PUBLIC_HTTPS_AND_CERT_EXPIRY_READONLY_20260702_PROOF.txt\n\n## NPMPLUS_ROUTE_CERT_MAPPING_READONLY_20260702\n- Read-only NPMplus route to certificate mapping audit completed.\n- Expected proxy routes were found and expected certificate files are present and valid for more than 30 days.\n- No certbot renew/dry-run was executed.\n- Proof: /root/evidence/687_NPMPLUS_ROUTE_CERT_MAPPING_READONLY_20260702_PROOF.txt\n\n## TODAY_PROOFS_SECRET_SCAN_READONLY_20260702\n- Read-only filename-only secret pattern scan completed for today proof files and active reference/error register.\n- No matching secret value patterns were found.\n- Proof: /root/evidence/688_TODAY_PROOFS_SECRET_SCAN_READONLY_20260702_PROOF.txt\n\n## HOMELAB_20260702_SESSION_CLOSURE_OK\n- Session closure proof created for final OK chain 677-688.\n- TaftAuto, Homepage, Cloudflare tokens, evidence map, cert hooks, deploy guards, public HTTPS, route/cert mapping, and secret scan are documented.\n- Proof: /root/evidence/689_HOMELAB_20260702_SESSION_CLOSURE_OK_PROOF.txt\n\n## CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702\n- edge-vm CrowdSec CAPI is registered and enabled through NetBird exit route.\n- Client peer: edge-vm.netbird.selfhosted / 100.100.60.182.\n- Primary exit routing peer: relay.netbird.selfhosted / 100.100.19.1 / Moldova.\n- Backup egress peer present: mail.netbird.selfhosted / 100.100.147.204 / USA.\n- NetBird exit route proof on edge-vm: wg allowed-ips includes 0.0.0.0/0 via relay; ip route get 1.1.1.1 uses wt0 table 7120.\n- External trace after route: 85.121.4.192 / OTP, not home 95.84.154.183 / ARN.\n- CrowdSec proof: cscli lapi status OK, cscli capi status OK, CAPI sharing/blocklist pull enabled, no DISABLE_ONLINE_API or -no-capi flags in active compose.\n- WARP is intentionally absent and must not be reintroduced for this path.\n- Unauthenticated https://api.crowdsec.net/v3/watchers/login returning HTTP 403 is expected network reachability proof.\n- Final proof: /root/evidence/771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_20260702T191057Z_PROOF.txt\n\n## HOMEPAGE_RU_LAYOUT_MONITORING_PROPOSALS_20260702\n- Homepage live config is on core-apps VM [PRIVATE_IP] under /opt/stacks/homepage/config.\n- Groups/descriptions were normalized to Russian.\n- Layout is row/6-columns for all active groups.\n- siteMonitor fields are preserved; monitors must be repaired, not removed.\n- Cloudflare card must be left untouched by explicit operator request.\n- Red badges after migration were primarily caused by Homepage container resolving *.vpn.gram1.ru to edge NetBird IP and timing out; fixed by mapping vpn hostnames to edge LAN [PRIVATE_IP] in Homepage compose extra_hosts.\n- Router Moscow monitor uses edge health endpoint http://[PRIVATE_IP]:18091/router-moscow.\n- NPMplus monitor uses edge health endpoint http://[PRIVATE_IP]:18091/npmplus.\n- Missing-card proposal report: /var/lib/homelab-health/homepage-missing-card-proposals-final.txt.\n\n## UPTIME_KUMA_MISSING_MONITOR_AND_PROPOSALS_20260702\n- Uptime Kuma lives on monitoring VM [PRIVATE_IP].\n- Added one missing monitor: NPMplus Edge Health -> http://[PRIVATE_IP]:18091/npmplus.\n- Uptime Kuma DB backup was created before DB mutation under /root/uptime-kuma-manual-backups/773a-* on monitoring VM.\n- DB integrity after insertion was OK.\n- Deep monitoring proposal report: /var/lib/homelab-health/uptime-kuma-monitoring-proposals.txt.\n- Cloudflare must not be touched by operator request.\n- Recommended next monitors are P0 only after operator approval: Router Moscow Health, Backup/Restore Dashboard, Restore Drill Index, NetBird Peers, CrowdSec CAPI, Paperless Restore.\n\n## UPTIME_KUMA_NPMPLUS_ADMIN_REPAIR_20260702\n- Existing Uptime Kuma monitor \"NPMplus Admin\" was red because it checked https://[PRIVATE_IP]:81/.\n- NPMplus admin is intentionally localhost-bound on edge-vm, so monitoring VM should not check the admin UI directly.\n- Fixed monitor target to edge health endpoint: http://[PRIVATE_IP]:18091/npmplus.\n- Monitor was repaired, not deleted.\n- Cloudflare was not touched.\n\n## UPTIME_KUMA_APPROVED_MONITOR_BATCH_20260702\n- Approved Uptime Kuma monitor batch installed after Homepage/Kuma analysis.\n- Existing NPMplus Admin monitor was repaired to http://[PRIVATE_IP]:18091/npmplus, not deleted.\n- Edge health wrapper /usr/local/sbin/router-moscow-health provides safe endpoints on [PRIVATE_IP]:18091 for router-moscow, npmplus, loki, alloy, cadvisor, registry-cache, socket-proxy, diun, kopia and crowdsec.\n- Cloudflare was not touched by explicit operator request.\n- Uptime Kuma DB backups are under /root/uptime-kuma-manual-backups/773d-approved-batch-* on monitoring VM before the DB mutation.\n\n## UPTIME_KUMA_PENDING_HEALTH_MONITORS_FIX_20260702\n- Four pending Uptime Kuma keyword monitors were repaired by using local pve01 health endpoint paths instead of public backup.gram1.ru paths:\n - Backup Restore Dashboard -> http://[PRIVATE_IP]:9101/backup-restore-dashboard.txt\n - Restore Drill Index -> http://[PRIVATE_IP]:9101/restore-drill-index.txt\n - Paperless Restore -> http://[PRIVATE_IP]:9101/paperless-restore.txt\n - AdGuard Rewrite Sync -> http://[PRIVATE_IP]:9101/adguard-rewrite-sync.txt\n- Cloudflare was not touched.\n- Monitors were repaired, not deleted.\n\n## UPTIME_KUMA_FOUR_HEALTH_SOURCES_REPAIRED_20260702\n- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR.\n- Repaired pve01 health source files under /var/lib/homelab-health:\n - backup-restore-dashboard.txt\n - restore-drill-index.txt\n - paperless-restore.txt\n - adguard-rewrite-sync.txt\n- Kuma monitors point to local pve01 health endpoints on http://[PRIVATE_IP]:9101/.\n- Cloudflare was not touched.\n- Monitors were repaired, not deleted.\n\n## DOCKGE_DISCONNECTED_HOSTS_AND_STALE_STACKS_20260702\n- Dockge is currently present only on edge-vm.\n- core-apps and monitoring run Docker containers but have no Dockge/Dockge-agent detected.\n- Dockge \"2 not connected\" should be interpreted as two disconnected Docker hosts unless later evidence shows otherwise.\n- No stopped containers were found across edge-vm, core-apps and monitoring during analysis.\n- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring.\n- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification.\n\n## DOCKGE_REMOTE_AGENTS_INSTALLED_20260702\n- Dockge main instance runs on edge-vm [PRIVATE_IP].\n- Remote Dockge agents were installed on:\n - core-apps [PRIVATE_IP]:5001\n - monitoring [PRIVATE_IP]:5001\n- Edge Dockge agent table stores the two agent URLs with username/password; password must never be printed.\n- No runtime containers were deleted.\n- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first.\n\n## DOCKGE_REMOTE_AGENTS_AND_STALE_ARCHIVE_FINAL_20260702\n- Dockge main instance: edge-vm [PRIVATE_IP].\n- Remote agents connected: core-apps [PRIVATE_IP]:5001, monitoring [PRIVATE_IP]:5001.\n- Stale moved edge stack folders archived under /opt/dockge-stale-archive/20260702T230442Z on edge-vm; no runtime containers were deleted.\n- External remote stacks exposed to Dockge using bind mounts: /opt/vaultwarden-compose, /opt/gotify-compose, /opt/uptime-kuma-compose into /opt/stacks.\n- Final target: remote not_visible count must be 0 and edge stale remaining matches must be 0.\n\n## DOCKGE_FINAL_MANAGED_STACKS_20260702\n- Dockge main stack, npmplus and remote dockge-agent stacks were recreated from /opt/stacks so Dockge can manage them.\n- Edge Dockge listens on 127.0.0.1:5001; LAN probe to [PRIVATE_IP]:5001 may return 000 and is not the correct health proof.\n- Remote agents listen on [PRIVATE_IP]:5001 and [PRIVATE_IP]:5001.\n- Stale moved stacks were archived, not deleted.\n\n## SERVICE_SETUP_ROADMAP_DEEP_20260702\n- Configuration order: secrets, DNS/proxy/TLS, backup/restore, monitoring, SSO, docs, core data apps, sensitive apps, productivity, media, automation, utilities.\n- Do not configure data-heavy or automation services before backup and monitoring are proven.\n- Per-service done criteria: URL, TLS, admin, SSO decision, volumes, backup, restore test, Kuma, Homepage, docs.\n\n## PHASE00_FREEZE_CLOSED_20260702\n- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain.\n- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml.\n- Old systemd unit homelab-registry-cache.service was disabled and masked because it recreated the container without Compose labels.\n- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/.\n- Next phase: Vaultwarden and break-glass secret baseline.\n\n## PHASE00_FREEZE_CLOSED_20260702\n- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain.\n- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml.\n- Old systemd unit homelab-registry-cache.service was archived and masked because it recreated the container without Compose labels.\n- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/.\n- Next phase: Vaultwarden and break-glass secret baseline.\n\n## PHASE00_FREEZE_CLOSED_20260702\n- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain.\n- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml.\n- Old systemd unit homelab-registry-cache.service was archived and masked with /dev/null because it recreated the container without Compose labels.\n- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/.\n- Next phase: Vaultwarden and break-glass secret baseline.\n\n## OPERATOR_RULE_CLOSE_TAILS_IMMEDIATELY_20260714\n- Жёсткое правило: хвосты не оставлять. Любая проблема, временный артефакт, неудачная проверка, блокер или анти-регрессия, обнаруженные в текущем scope, должны быть устранены и закрыты до перехода к следующей задаче.\n- После любой неуспешной команды текущая задача остаётся активной до установления точной причины, исправления, проверки исправления, добавления анти-регрессии, очистки временных артефактов и выпуска закрывающего proof.\n- Этап разрешено помечать CLOSED только при UNRESOLVED_TAIL_COUNT=0, BLOCKER_COUNT=0, TEMPORARY_ARTIFACT_COUNT=0, проверенном rollback, зелёном health, обновлённых proof и reference.\n- Запрещено откладывать выявленный хвост только ради удобства или перехода к следующему этапу.\n- Если технически необходим отдельный подэтап, он должен быть ограничен точным scope, выполнен и закрыт немедленно до возврата к основной работе.\n- Исключения: внешняя зависимость, опасная неоднозначность, риск раскрытия секрета, отсутствующий обязательный файл или явное решение пользователя. В таком случае статус должен быть BLOCKED или OPEN с точным блокером; статус CLOSED запрещён.\n- Следующий этап не начинается, пока текущий хвост не закрыт.\n- Proof: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/report.txt\n- Secret scan: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/strict-secret-scan.txt\n\n## CLUSTER_ADMIN_STAGE4H_SCOPE_DEFINED_20260714\n- Stage4G остаётся закрытым и стабильно работает в режиме observe-notify.\n- Закрыты два хвоста dependency-аудита: глобальный LIMIT всего UNION ALL и узкий шаблон collector_patch_required.\n- Migration plan содержит collector_patch_required=true; dynamic contract содержит collector_patch_required_after_stage4c=true. Это два подтверждения одного обязательного требования.\n- Migration 003 не применена, 12 новых столбцов отсутствуют.\n- Migration выполняет полный UPDATE observations, четыре SET NOT NULL, пять VALIDATE CONSTRAINT и пять неконкурентных CREATE INDEX.\n- Текущий collector не формирует обязательный provenance-набор; готовый collector patch отсутствует.\n- Production adapters и production apply phase отсутствуют.\n- Отдельный apply migration 003 без collector patch запрещён.\n- Безопасный порядок: collector patch candidate → временная БД и acceptance/rollback → контролируемый migration+collector cutover → adapter integration.\n- Текущий этап открыт: 4H_COLLECTOR_PROVENANCE_PATCH_AND_MIGRATION003_PREAPPLY_READINESS.\n- Предыдущие хвосты закрыты; Stage4H не считается CLOSED до реализации и полного seal.\n- Proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt\n- Secret scan: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/strict-secret-scan.txt\n\n## CLUSTER_ADMIN_STAGE4H_COLLECTOR_BASELINE_AUDIT_CLOSED_20260714\n- Исправленный baseline-аудит collector завершён.\n- collector.py: 113 строк, SHA256 подтверждён.\n- Привилегированные файлы нельзя читать через caller-side `< file` перед sudo; применён sudo wc без внешнего перенаправления.\n- Фактические поля collector_runs: finished_at и error_text; completed_at и error отсутствуют.\n- Найдены один canonical source instance и один collector_runs_foundation_enrich_trg.\n- Trigger обогащает collector_runs полями source_instance_id и run_key.\n- Текущий collector ещё не пишет provenance-поля observations; migration 003 не применена.\n- Production осталась 0|0|OK, timer активен, failed units отсутствуют.\n- Baseline-аудит закрыт без хвостов; Stage4H остаётся OPEN для isolated collector provenance patch candidate.\n- Proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt\n- Secret scan: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/strict-secret-scan.txt\n\n## CLUSTER_ADMIN_STAGE4H_COLLECTOR_PATCH_CANDIDATE_CLOSED_20260714\n- Из точного live collector создан isolated provenance patch candidate; production collector не изменён.\n- Candidate пишет все 12 migration003 provenance-полей и применяет SHA256 fallback-дедупликацию.\n- Metadata разрешает только instance, job, mountpoint и device, ограничена по типам и размеру.\n- Candidate скомпилирован локально и проверен на VM180 Python 3.11.2 от clusteradmin.\n- SSH stdin harness исправлен: bash -s получает ровно path, SHA256 и bytes; первый аргумент дополнительно проверяется по безопасному шаблону.\n- Self-test и отрицательный CLI-тест RC64 прошли; временный remote-файл удалён.\n- Production осталась 0|0|OK; live collector и timer не изменены.\n- Неудачная попытка закрыта без хвостов; Stage4H остаётся OPEN до temporary-DB acceptance и controlled cutover.\n- Candidate root: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z\n- Proof: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/report.txt\n- Secret scan: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/strict-secret-scan.txt\n\n## OPERATOR_RULE_NO_OVERSIZED_MONOLITH_COMMANDS_20260714\n- Запрещены чрезмерно длинные интерактивные однострочные команды с глубокой вложенностью SSH, SQL, Python, awk и кавычек.\n- Рекомендуемый предел интерактивной команды: 8000 байт. Превышение допускается только для простого текста без вложенных языков.\n- Сложная операция оформляется как отдельный versioned task-скрипт с contract, syntax-check, dry-run, manifest, rollback и proof.\n- Создание task-скрипта и его запуск выполняются разными короткими командами.\n- Remote stdout/stderr всегда сохраняются до проверки RC; запрещено терять вывод из-за errexit-sensitive command substitution.\n- Перед запуском проверяются SHA256, размер, владелец, режим и синтаксис task-скрипта.\n- Обрезанная, повреждённая или частично вставленная команда считается невыполненной; сначала проводится side-effect audit.\n- Хвост parse-ошибки закрывается только после доказательства отсутствия изменений и временных артефактов.\n- Proof: /var/lib/homelab-change-sets/operator-command-size-guard-v1/20260714T182653Z/report.txt\n\n## HOMELAB_ADMIN_CLI_CONTRACT_20260714\n- Поддерживаемые команды: version, status, lint TASK_DIR, validate TASK_DIR, run TASK_DIR PHASE.\n- Отдельная команда --help отсутствует. Неизвестный аргумент печатает usage и штатно возвращает RC=64.\n- Нельзя трактовать RC=64 от --help как неисправность runner.\n- Для проверки доступности runner использовать homelab-admin version; для структуры CLI использовать зафиксированную usage-строку.\n- Proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v1/20260714T183338Z/report.txt\n\nAUTOMATIC_CLUSTER_CONTEXT_REFRESH_20260721\nCollector: /usr/local/sbin/homelab-context-collect\nScheduler: hourly at minute 17 via /etc/cron.d/homelab-context-refresh\nDestination: immutable Gitea runtime history with CONTEXT-AUTO UTC command IDs.\n\nAUTOMATIC_CLUSTER_CONTEXT_REFRESH_DIRECT_20260721\nPublisher: /usr/local/sbin/homelab-context-refresh-direct\nSchedule: hourly at minute 17.\nDestination: immutable assistant-runtime history with CONTEXT-AUTO UTC IDs.\nDETAIL=SECTION_REFERENCE_REGISTER_END\nDETAIL=SECTION_ERROR_REGISTER_BEGIN\n# HOMELAB ASSISTANT ERROR REGISTER\n\nНазначение: перед каждой следующей командой сверяться с этим файлом.\n\n## Критические ошибки ассистента\n1. Повторно дал слишком большой интерактивный paste в shell.\n2. Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт.\n3. Дал генератор справочника прямо в терминал вместо безопасного маленького шага.\n4. Нарушил своё же правило: не давать длинные вложенные команды с кавычками.\n\n## Жёсткие правила перед каждой командой\nCHECK-1: команда не должна быть большим paste.\nCHECK-2: команда не должна содержать большой here-doc.\nCHECK-3: команда не должна смешивать Markdown, backticks и shell-логику.\nCHECK-4: команда не должна иметь вложенный ssh с несколькими уровнями кавычек.\nCHECK-5: команда не должна печатать секреты.\nCHECK-6: если создаётся файл, сначала маленький безопасный шаг, потом проверка.\nCHECK-7: если команда длиннее 8 строк, её нельзя давать в интерактивный shell.\nCHECK-8: для edge-vm использовать debian@[PRIVATE_IP] и sudo.\nCHECK-9: для forum-prod использовать pve02 и ключ [SENSITIVE_PATH]\nCHECK-10: Corosync не трогать без отдельного плана и rollback.\n\n## Текущие важные факты\nInternal network: [PRIVATE_IP]/24.\nMigration config: migration: secure,network=[PRIVATE_IP]/24.\nCorosync remains on [PRIVATE_IP]/12/13.\nVM160 forum-prod is not in Proxmox nightly backup.\nVM130 edge-vm has secondary disk backup=0 risk.\n05_edge_compose_safe.tgz quarantined.\n\n## Правило для справочника\nНе генерировать большой справочник через интерактивную вставку.\nСледующий справочник делать только маленькими append-блоками или через файл, созданный вне интерактивного shell.\n\n11. Ошибка: считать offhost OK после failed rsync.\nЕсли rsync/scp упал, нельзя проверять latest offhost-файл без сверки имени.\nПроверка должна подтвердить именно новый архив, например дату 2026_06_30-00_20_59.\nСтарый OFFHOST_ZSTD_OK не закрывает новый backup.\n\n12. Ошибка: широкий secret-поиск по /opt/stacks дал шум.\nНе искать Cloudflare/SMTP сразу по /opt/stacks, MkDocs site, backups и HTML-отчётам.\nДля ротации сначала смотреть live-конфиги: /opt/npmplus/tls/certbot/renewal/*.conf и реальные credentials-файлы из них.\nЗначения секретов не печатать; выводить только пути, ключи и redacted-поля.\n\n13. Ошибка: SQL с одинарными кавычками внутри одинарной SSH-команды ломается.\nНельзя писать sqlite SQL вида j.type in ('object','array') внутри ssh '...'.\nДля sqlite через ssh использовать SQL без одинарных кавычек: char(36), length(j.atom), двойные внешние кавычки, либо отдельный файл.\nПеред запуском проверять, что команда не содержит конфликтующих уровней кавычек.\n\n14. Ошибка: путать контекст входа и узел выполнения.\nСтартовая точка оператора: root@pve01 / [PRIVATE_IP].\nedge-vm: ssh debian@[PRIVATE_IP], внутри использовать sudo.\npve02/pve03: ssh root@pve02 или ssh root@pve03.\nforum-prod: заходить через pve02, ключ [SENSITIVE_PATH]\nПеред каждой командой явно понимать: где выполняется локальная часть, где remote-часть, каким пользователем.\n\n15. Ошибка: повторно нарушено правило №13 после его добавления.\nСнова был использован SQL JSON-path в одинарных кавычках внутри ssh '...'.\nЗапрещено использовать в remote sqlite SQL фрагменты вида '$.key' или '.key'.\nДля JSON-path использовать только char(...), без одинарных кавычек внутри SQL.\nКоманду с ошибкой char(36)||.dns_provider считать битой и не использовать.\n\n16. Основополагающее правило: перед каждой командой ассистент обязан явно написать: Сверка с файлом ошибок: CHECK пройден.\nПосле этой строки можно давать только одну короткую команду или один логический блок команд.\nНельзя выдавать команды без предварительной сверки с этим файлом ошибок.\nНельзя продолжать после собственной ошибки без записи ошибки в этот файл.\n\n19. Уточнение формата: не писать отдельную строку сверки перед каждой командой в пакете.\nСверка должна быть внутри самой команды первым логическим блоком: grep правил из файла ошибок и RULE_CHECK_OK.\nРазрешено давать до трёх логически связанных команд сразу, по возможности в одну строку каждая.\nНе писать фразу \"ждём\" после команд; указывать только контрольные строки результата.\n\n20. Ошибка: широкий XenForo SMTP-поиск по src дал шум исходников.\nДля XenForo SMTP не grep-ать весь /src; сначала смотреть live src/config.php без значений, затем БД XenForo и xf_option/emailTransport.\nЗначения DB-паролей и SMTP-паролей не печатать.\n\n21. Ошибка: nested PHP php -r дал Parse error на forum-prod.\nКоманды 102-104 считать битыми: они не подтвердили и не изменили smtpSsl.\nНе продолжать длинные вложенные ssh+php -r с кавычками; для forum-prod лучше войти в VM или использовать короткие mysql/php команды без вложенного кода.\nРабочими считать: DB backup перед SMTP, SMTP password update, JSON postcheck; SMTP auth ещё не закрыт из-за CODE=530 на AUTH LOGIN.\n\n22. Ошибка: самодельный base64 PHP для SMTP auth сломан.\nКоманда 108 дала PHP Parse error на smtpHost и пустой proof-файл.\nНе использовать больше сгенерированные большие base64 PHP для SMTP/XenForo.\nДля XenForo SMTP использовать короткие mysql-запросы к xf_option и готовые штатные механизмы XenForo.\n\n23. Ошибка: MariaDB не поддержала cast('true' as json) в JSON_SET.\nКоманда 111 не изменила smtpSsl: OLD_SMTPSSL=false, SQL ERROR 1064, NEW_SMTPSSL=false.\nПеред боевым UPDATE сначала проверять JSON_SET синтаксис на тестовом JSON через SELECT.\nНе считать PHP_FPM_RELOAD_OK подтверждением изменения БД.\n\n24. Ошибка: exit 1 в interactive-check закрыл SSH-сессию.\nНельзя использовать { echo RULE_CHECK_FAIL; exit 1; } в командах, выполняемых прямо в интерактивном root@pve01 shell.\nПри failed-check использовать безопасный шаблон: if grep ...; then echo RULE_CHECK_OK; ...; else echo RULE_CHECK_FAIL; fi.\nНе делать fragile grep по точной строке с кавычками, если в файле ошибок строка содержит shell/SQL quoting.\n\n25. Ошибка анализа: не надо слепо включать smtpSsl=true при smtpPort=587.\nsmtpPort=587 обычно означает STARTTLS, а не implicit SMTPS.\nCODE=530 на AUTH LOGIN означает, что ручной checker не сделал STARTTLS; это не доказывает неверный SMTP-пароль.\nСначала проверять AUTH через openssl s_client -starttls smtp, без вывода секрета.\n\n26. Ошибка: openssl -crlf вместе с ручным CRLF ломает SMTP AUTH checker.\nКоманда 116 получила 535 Invalid base64 data in continued response после 334 Username.\nЭто указывает на битую base64-строку checker-а, а не на неверный SMTP-пароль.\nНе использовать -crlf, если команды уже отправляются с явным \\r\\n.\nДля 587 оставлять smtpSsl=false и проверять STARTTLS корректным checker-ом.\n\n27. Ошибка: Python heredoc внутри ssh сломал not_ok диагностику.\nКоманда 138 пересобрала dashboard, но диагностический Python-блок упал с NameError из-за кавычек.\nНе использовать heredoc Python внутри вложенного ssh; для dashboard verification использовать cat/grep JSON или простые команды.\n\n28. Ошибка: docker compose ls --format json | wc -l дал ложный COMPOSE_PROJECT_COUNT=1.\nПричина: JSON-вывод может быть одной строкой/структурой, wc -l не является счётчиком проектов.\nДля Docker Compose count использовать обычный docker compose ls --all + awk по строкам таблицы или JSON parser, не wc -l.\n\n29. Ошибка: monitoring compact status искал неверные имена health-файлов.\nФакт: npmplus-cert-expiry.txt не является подробным файлом; подробный файл называется npmplus-certificate-expiry.txt, а статусный alias npmplus-cert-expiry-health.txt.\nФакт: health-metrics.txt не обязан существовать как health-файл; health metrics могут публиковаться через node-exporter textfile collector.\nДля Prometheus targets проверять HTTP_CODE/RC явно, не оставлять пустой блок PROMETHEUS_TARGETS_LOCAL.\n\n30. Ошибка/инцидент: Prometheus API HTTP_CODE=000 после monitoring reference.\nФакт: в monitoring compact status curl к 127.0.0.1:9090 вернул HTTP_CODE=000, а raw inventory не показал контейнер prometheus среди monitoring containers.\nНельзя считать monitoring слой runtime-closed только по secret scan/reference proof; нужно отдельно проверить Prometheus container/API и обновить справочник по факту.\n\n31. Строгое правило: перед каждой командой сверяться с файлом ошибок и справочником.\nПеред любым действием команда обязана проверять /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md и текущий 31_HOMELAB_REFERENCE.md.\nКоманда обязана печатать ERROR_REGISTER_CHECK=OK и REFERENCE_CHECK=OK до основного действия.\nЕсли сверка не пройдена, команда не должна выполнять основное действие. Не использовать exit 1, чтобы не закрывать SSH-сессию.\n\n32. Ошибка: 153_PROMETHEUS_TARGETS_SETTLED_PROOF содержит SyntaxError из-за вложенного Python внутри SSH.\nФакт: Python -c внутри ssh потерял кавычки вокруг /tmp/prom-targets-settled.json, data, activeTargets, labels, job, health.\nФайл 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt нельзя считать валидным settled proof.\nДля таких проверок не использовать вложенный Python; использовать curl + grep/sed/awk либо отдельный локальный файл-скрипт.\n\n34. Ошибка операционного ввода: полный терминальный лог MobaXterm попал в shell.\nФакт: bash пытался выполнить строки Authenticating, MobaXterm banner, RULE_CHECK_OK, PROOF_OK и таблицы вывода как команды.\nТакие выводы не являются инфраструктурным состоянием; считать невалидными только соответствующие failed command attempts.\nДальше давать короткие команды и не вставлять обратно полный transcript в shell.\n\n33. Security finding: root authorized_keys на PVE-нодах имел права 777.\nФакт: inventory 158 показал [SENSITIVE_PATH] и backup-файлы с режимом 777 на pve01/pve02/pve03.\nНужно исправлять на [SENSITIVE_PATH]=700 и authorized_keys/authorized_keys.bak*=600, затем фиксировать before/after proof.\n\n35. Ошибка проверки: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED не доказал исправление прав.\nФакт: AFTER всё ещё показал 777 для [SENSITIVE_PATH] на pve01/pve02/pve03.\nВозможная причина: authorized_keys является symlink, а stat без -L показывает режим ссылки 777.\nНужно проверять stat -L целевого файла и считать 160 недостаточным proof.\n\n36. Quality check: Storage block needs integrity and pve03 capacity coverage review.\nФакт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.\nФакт: pve03 /mnt/staging observed at 77%, close to WARN threshold 80%, while disk-space.txt did not visibly list pve03_staging.\nBefore closing storage layer, verify block integrity and pve03 capacity coverage.\n\n37. Coverage gap: pve03_staging missing from disk-space health coverage.\nФакт: 169 showed pve03 /mnt/staging at 77%, close to WARN 80%, while disk-space.txt listed pve01_staging, pve02_staging and edge_immich_media only.\nBefore closing storage fully, add or document pve03_staging capacity monitoring.\n\n38. Quality check: Service Dependency Map block needs integrity review.\nФакт: terminal output around command 222 showed paste artifact near \"FAIL; fi\".\nBefore closing Service Dependency Map, verify 178_REFERENCE_SERVICE_DEPENDENCY_MAP_BLOCK.txt and the reference section for shell/log garbage or truncation.\n\n39. Ошибка проверки: post deep audit BAD_MARKER scan caught prose false positive.\nФакт: строка справочника с фразой No unresolved final REVIEW_REQUIRED... SyntaxError... была принята за BAD_MARKER.\nЭто не runtime/log мусор, а документированная фраза deep audit closure. В scan нужно исключать эту prose-строку.\n\n40. Ошибка/вывод: VM150 Mail cloud single-file upload failed with 413.\nФакт: Mail/WebDAV rejected one 6.6G vzdump file with 413 Request Entity Too Large.\nРешение: use VM150 recurring chunked script with 512M parts and download verification.\n\n41. Ошибка: post-backup-pass-slo-reconciliation checked pve01 router health on edge-vm.\nФакт: first reconciliation wrote STATUS=ERROR because router-running-config-mail-cloud.txt is on pve01, not edge-vm.\nFix: build reconciliation from pve01 and edge-vm health files by correct owner.\n\n## ASSISTANT_COMMAND_BATCHING_RULE_20260630\n- Rule: combine related infrastructure checks, proof generation, reference updates and validation scans into one safe batch command when practical.\n- Do not split work into many small commands when the operations are low-risk, read-only, or naturally part of one evidence block.\n- Still keep mandatory preflight checks, short bounded commands, no large paste, no here-docs, no interactive shell traps, and no sensitive values in output.\n- Split into smaller commands only when safety, rollback, quoting risk, or troubleshooting clarity requires it.\n\n## HOME_PORTAL_BASE64_APPLY_FAILURE_20260630\n- Prior home-portal apply attempt 312 failed because a large base64 Python payload decoded as invalid UTF-8.\n- Rule: avoid large opaque base64 script payloads for homelab changes; use transparent bounded shell/perl/python commands instead.\n\n## HOMELAB_COMMAND_SAFETY_HARDENING_20260630\n- Rule: do not use large opaque payloads, generated base64 scripts, or hidden multi-line script blobs for homelab changes.\n- Use transparent bounded shell/perl/python commands, or create a temporary readable script and validate it before execution.\n- Any script used for changes must pass a syntax/parse check before it is allowed to modify files or restart services.\n- Any Traceback, UnicodeDecodeError, SyntaxError, Permission denied, command-not-found, nonzero critical RC, or failed write marker must force REVIEW or FAILED, never OK.\n- Success requires both runtime health and content-specific post-checks: expected files changed, expected cards exist, duplicates absent, service/container alive, and no new critical log pattern.\n- HTTP 200 alone is not a success condition for configuration changes.\n- If an apply step fails, the next command must explicitly record the failed attempt and then run a corrected transparent apply command.\n\n## HOME_PORTAL_API_ERROR_CLOSURE_RULE_20260630\n- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200.\n- If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed.\n- External informational widgets such as weather/Open-Meteo must not block the service launcher portal.\n\n## HOME_PORTAL_COMPLEX_QUOTING_FAILURE_20260630\n- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('.\n- Cause: command was too complex and fragile due to nested shell/perl/python quoting.\n- Rule: do not use deeply nested one-liners for config edits; first run short state checks, then apply small transparent edits.\n\n## HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RULE_20260630\n- Rule: Homepage cleanup validation must check active config files only, not logs or backup files.\n- Active files are services.yaml, bookmarks.yaml, widgets.yaml, settings.yaml and docker.yaml under /opt/stacks/homepage/config.\n- Matches in /logs/ or *.bak* are historical evidence only and must not make cleanup REVIEW unless the active config still contains the target.\n\n## HOME_PORTAL_LINK_VALIDATION_NPMPLUS_DEFAULT_RULE_20260630\n- Rule: Homepage link validation must not treat HTTP 200 as success if the body is the NPMPlus default “Congratulations” / host-not-set-up page.\n- Green siteMonitor dots can be false positives when NPMPlus returns its default vhost page.\n- A portal card is openable only when DNS, reverse-proxy host mapping, and target content are all valid.\n\n## ROUTER_CLI_STDIN_APPLY_FAILURE_20260630\n- Context: attempted Netcraze router ACL apply through SSH stdin/multiline for Homepage Moscow Router monitor fix.\n- Evidence: 391_MOSCOW_ROUTER_EDGE_VM_WEBADMIN_ACL_APPLY_PROOF.txt.\n- Failure: STDIN_SHOW_OK=no, APPLY_SKIPPED=stdin_show_failed, CONFIG_HAS_EDGE_5080=no, CONFIG_HAS_EDGE_5083=no.\n- Rule: do not apply Netcraze config through stdin/multiline SSH; prove CLI input mode first.\n\n## ROUTER_CLI_PROBE_STDIN_CONSUMPTION_ERROR_20260630\n- Context: ACL syntax read-only probe loop executed only one command because ssh consumed the loop stdin.\n- Evidence: 393_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_PROOF.txt showed COMMAND_COUNT=1.\n- Fix pattern: use ssh -n or redirect SSH stdin away from the command-list loop for all future SSH-in-loop probes.\n\n## ROUTERBACKUP_READONLY_CLI_PERMISSION_LIMIT_20260630\n- Context: routerbackup user can run show running-config but cannot execute configuration commands such as access-list.\n- Evidence: 394_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_FIXED_PROOF.txt.\n- Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli].\n- Rule: do not attempt ACL/config changes as routerbackup; use admin/operator router access or router Web UI for write changes.\n\n## PROOF_SUMMARY_EXTRACTION_BLANK_20260630\n- Context: proof 395 tried to read COMMAND_COUNT/ZERO_RC_COUNT from proof files 393/394, but those summary fields were printed to console and were not persisted inside the proof files.\n- Evidence: 395_ROUTER_CLI_ERROR_REGISTER_UPDATE_PROOF.txt has blank P393_COMMAND_COUNT/P394_COMMAND_COUNT/P394_ZERO_RC_COUNT fields.\n- Rule: when creating follow-up proof summaries, derive counts from persisted proof body lines such as CMD_INDEX and CMD_RC, or write summary fields into the proof file itself.\n\n## ROUTER_ACL_UI_RULE_DELETE_AND_RESTORE_20260701\n- Context: while moving Moscow Router monitor rules to _WEBADMIN_Bridge1, two _WEBADMIN_Bridge0 Nextcloud Talk TURN rules were accidentally removed.\n- Deleted/restored rules: Home [PRIVATE_IP]/24 to Nextcloud [PRIVATE_IP] TCP/UDP 3478.\n- Verification proof: 399_ROUTER_ACL_RESTORE_AFTER_UI_PROOF.txt.\n- Rule: after manual router Web UI ACL edits, compare _WEBADMIN_Bridge0, _WEBADMIN_Bridge1 and _WEBADMIN_GigabitEthernet1 before continuing.\n\n## ROUTER_MONITOR_NODE_LOOKUP_PROBE_BUG_20260701\n- Context: proof 400 tested Moscow router HTTPS monitor from Homepage container with custom Node lookup callback.\n- Failure: NODE_HTTPS_RESOLVE_ERROR=ERR_INVALID_IP_ADDRESS and NODE_HTTPS_RESOLVE_INSECURE_ERROR=ERR_INVALID_IP_ADDRESS.\n- Impact: Node HTTPS resolve subtest is invalid; curl --resolve results remain valid.\n- Rule: do not rely on custom Node lookup callback until separately tested; prefer curl --resolve or simple direct Node HTTP checks.\n\n## ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701\n- Context: Moscow Router Homepage monitor after ACL fix.\n- Evidence: proofs 399, 400, 401.\n- Observed: edge-vm TCP to [PRIVATE_IP]:5080/5083 is OK, but router HTTP/HTTPS returns 403 for all tested paths.\n- Impact: Homepage siteMonitor cannot be made green via direct router HTTP URL until router web service allows the Bridge1/Proxmox/protected segment or an explicit safe monitor endpoint is used.\n- Rule: do not keep changing ACL for this symptom; ACL is already correct. Fix service access scope or use a dedicated monitor endpoint.\n\n## PY_COMPILE_PYC_PERMISSION_ERROR_20260701\n- Context: installing edge-vm Moscow router health endpoint.\n- Mistake: assistant used `python3 -m py_compile` on a script installed under /usr/local/sbin as root.\n- Actual impact: non-root syntax check attempted to create /usr/local/sbin/__pycache__ and failed with Permission denied.\n- Recovery: validate syntax with `ast.parse` instead of py_compile, because it does not write .pyc files.\n- Rule: do not use py_compile against root-owned system paths from an unprivileged user.\n\n## ROUTER_HOME_IP_DOCKER_ROUTE_CONFLICT_20260701\n- Context: Moscow Router Homepage siteMonitor attempt using http://[PRIVATE_IP]:5080.\n- Evidence: proof 405 showed route to [PRIVATE_IP] via Docker bridge and Homepage container ECONNRESET.\n- Impact: do not use [PRIVATE_IP] directly as Homepage siteMonitor from edge-vm.\n- Mitigation: use dedicated edge-vm health endpoint that checks router TCP [PRIVATE_IP]:5080 and returns HTTP 200/503 for Homepage.\n\n## HOMEPAGE_ROUTER_MOSCOW_YAML_TITLE_SHAPE_MISMATCH_20260701\n- Context: applying Moscow Router Homepage siteMonitor health endpoint.\n- Mistake: assistant apply script matched only property-style cards with `title`, but active Homepage YAML can use service-name-key style like `- Service Name:`.\n- Evidence: proof 407 returned YAML_CHANGE_COUNT=0 and APPLY_SKIPPED=unexpected_match_count while health endpoint proof 406 was OK.\n- Recovery: use YAML-aware updater that handles both property-style and service-name-key style cards.\n\n## HOMEPAGE_ROUTER_MOSCOW_APPLY_SCRIPT_SYNTAX_ERROR_20260701\n- Context: applying Moscow Router Homepage health endpoint.\n- Mistake: assistant generated Python script with invalid f-string escaping.\n- Evidence: proof 408 showed SyntaxError in /tmp/homepage-router-moscow-apply-fixed.py.\n- Actual impact: YAML was not changed, so Homepage green dot could not appear.\n- Recovery: stop applying blind updater; first discover exact active YAML shape, then apply minimal YAML-aware edit.\n- Rule: validate generated Python syntax locally before scp/remote execution; avoid f-strings in generated maintenance scripts.\n\n## FORUM_PROD_VM160_FIRST_BOOT_SSH255_20260701\n- Context: clean rebuilt VM160 first boot.\n- Symptom: PING_OK=yes and TCP22_OK=yes, but SSH_RC=255.\n- Rule: do not proceed with OS baseline until SSH failure is diagnosed; likely old known_hosts key or cloud-init/root-key issue.\n\n## FORUM_PROD_VM160_NESTED_HOSTNAME_PROOF_QUOTING_20260701\n- Context: VM160 first SSH proof after rebuild.\n- Issue: command substitution $(hostname) inside nested ssh was expanded on pve02 before entering VM160.\n- Impact: VM_HOSTNAME=pve02 in proof 426 is not valid VM identity evidence.\n- Rule: for nested SSH identity checks, run literal hostname commands without local command substitution.\n\n## FORUM_PROD_VM160_SWAP_PROOF_QUOTING_20260701\n- Context: VM160 swapfile proof 429.\n- Issue: nested command substitution expanded on pve02, producing invalid SWAPFILE/FSTAB proof lines.\n- Impact: proof 429 is not valid closure evidence even though swap was active.\n- Rule: avoid nested $(...) in VM SSH proofs; use literal remote commands and clean proof.\n\n## FORUM_UPLOAD_ARCHIVES_VM_VERIFY_QUOTING_20260701\n- Context: proof 446 copy/check archives inside VM160.\n- Issue: nested SSH quoting expanded shell variables incorrectly, producing gzip checks against empty .gz and blank TAR_TOP lines.\n- Impact: SCP_TO_VM_RC=0 and file size list are valid, but GZIP/TAR proof lines in 446 are invalid.\n- Rule: verify VM archive integrity locally inside forum-prod, not through nested quoted loops.\n\n## FORUM_PROD_BULK_IMPORT_PHP85_EMPTY_FRONTEND_20260701\n- Context: rebuilt VM160, five XenForo forums imported in bulk on PHP 8.5.7.\n- Symptom: all five forums returned HTTP 200 with empty body; XenForo logged Template public:PAGE_CONTAINER/forum_list is unknown.\n- Additional failed approach: proof 480 ran xf-dev:rebuild-caches on CodeVipe and returned RC=1, not a valid fix.\n- Decision: stop bulk repair, treat current VM160 as lab, find a working CodeVipe recipe first, then destroy/rebuild cleanly for all five forums.\n- Rule: no more mass rebuild/repair commands across all forums until single-forum CodeVipe recipe is proven.\n\n## XENFORO_REQUIREMENTS_DIRECT_DOWNLOAD_403_20260701\n- Context: Tried to download XenForo requirements ZIP directly from xenforo.com inside forum-prod.\n- Issue: curl returned HTTP 403; unzip then failed because the ZIP was not downloaded.\n- Evidence: proof 491 showed DOWNLOAD_RC=22, ZIP_LIST_RC=9, UNZIP_RC=9.\n- Impact: proof 491 is not a valid server compatibility test.\n- Rule: use operator-uploaded xenforo23-requirements-test.zip from /root/forum-upload instead of relying on direct curl download.\n\n## PVE01_UNZIP_MISSING_FOR_XF_REQUIREMENTS_VERIFY_20260701\n- Context: proof 492 tried to inspect uploaded xenforo23-requirements-test.zip on pve01.\n- Issue: pve01 does not have unzip installed, so ZIP_TEST_RC=127 was not a ZIP integrity result.\n- Impact: proof 492 confirmed file presence only, not archive validity.\n- Rule: verify ZIP with Python zipfile or inside forum-prod rather than installing unzip on Proxmox just for inspection.\n\n## FRESH5_DEPLOY_SUCCESS_20260701\n- Context: VM160 was destroyed/recreated, then five XenForo forums were deployed from fresh ZIP backups.\n- Result: proof 513 confirms all five forums locally healthy.\n- Note: proof 511 initially failed dsmods path detection; proof 512 fixed dsmods by locating public_html via src/XF.php marker.\n- Rule: future restores should preserve internal_data/code_cache or rebuild compiled template cache before smoke testing.\n\n## NPMPLUS_SQLITE_PASTE_FAILURE_20260701\n- Context: NPMplus SQLite inspection command was too complex and a Python fragment was pasted into bash.\n- Issue: shell entered multiline prompt and produced syntax errors.\n- Impact: do not trust that SQLite inspection attempt.\n- Rule: avoid nested SSH heredoc/Python for this task; use simpler commands, generated nginx configs, or uploaded/local script files.\n\n## NPMPLUS_PUBLISH_SCRIPT_WRONG_ENV_SOURCE_525_20260701\n- Context: proof 525 tried to create forum proxy hosts in NPMplus.\n- Issue: script sourced /opt/npmplus/.env, but INITIAL_ADMIN_EMAIL/PASSWORD were not present there; they are in the npmplus container env.\n- Impact: no forum proxy hosts were created by proof 525.\n- Rule: read NPMplus API login values from docker inspect env internally, never print them.\n\n## NPMPLUS_API_LOGIN_MISSED_HTTPS81_AND_JSON_ESCAPE_20260701\n- Context: NPMplus API login attempts in proofs 526/527 failed.\n- Issue: commands missed the likely HTTPS admin endpoint on port 81 and built JSON by string interpolation instead of jq escaping.\n- Impact: no proxy hosts were created by 526/527.\n- Rule: for NPMplus API use jq-generated JSON and test https://127.0.0.1:81/api/tokens with -k before publish.\n\n## NPMPLUS_API_AUTH_UNAVAILABLE_MANUAL_ROUTES_20260701\n- Context: NPMplus API login attempts failed even though the forum VM and edge-to-forum connectivity are healthy.\n- Issue: API credentials from container initial env are not accepted by current NPMplus API.\n- Impact: do not use NPMplus API for this publish path.\n- Rule: publish forum routes through backed-up manual nginx proxy_host configs, then issue proper certificates after DNS points to edge.\n\n## EDGE_CERTBOT_TOKEN_FILE_PERMISSION_535_20260701\n- Context: proof 535 attempted DNS-01 certificate issue for five forum domains.\n- Issue: /tmp/forum_cf_token.env was root-owned mode 600, so debian user could not source it directly.\n- Impact: certbot ran without token and all certificate attempts failed; DNS cutover must not proceed from proof 535.\n- Rule: read Cloudflare token via sudo without printing it, then create temporary certbot credentials file and remove it after use.\n\n## EDGE_CERTBOT_FAILED_DNS_CUTOVER_ABORTED_20260701\n- Context: edge manual routes are working locally, but certbot DNS-01 certificate issue failed in proof 538.\n- Issue: all five certbot attempts returned RC=1; proof 539 correctly aborted DNS cutover because edge cert precheck was not ready.\n- Impact: Cloudflare DNS records were not changed by proof 539; public DNS still points to old IP 87.236.18.45.\n- Rule: inspect certbot logs and Cloudflare token zone permissions before retrying certificate issuance or DNS cutover.\n\n## CLOUDFLARE_TOKEN_NO_ZONE_ACCESS_20260701\n- Context: Cloudflare token discovered on edge verified as a token but could not access forum zones.\n- Evidence: proof 542 showed all five zones missing and DNS record create probes failed.\n- Impact: certbot DNS-01 failed and DNS cutover was correctly aborted; public DNS still points to old hosting.\n- Rule: install a new Cloudflare token with Zone Read and DNS Edit for the five forum zones before retrying certs/DNS.\n\n## FORUM_PUBLICATION_FINAL_SUCCESS_20260701\n- Context: five XenForo forums were rebuilt on VM160, routed through edge NPMplus, issued Let’s Encrypt certs using a corrected Cloudflare token, and cut over in DNS.\n- Result: final public proof 546 passed.\n- Important corrections: initial NPMplus API route failed; manual Nginx routes were used. Initial Cloudflare token had no zone access; corrected token passed zone/DNS probe.\n\n## FINAL_SNAPSHOT_NAME_TOO_LONG_556_20260701\n- Context: proof 556 final health gate passed for all five public forums.\n- Issue: Proxmox snapshot failed because generated snapshot name exceeded 40 characters.\n- Evidence: qm snapshot returned snapname value may only be 40 characters long.\n- Impact: forum health was OK, but proof 556 snapshot step was not completed.\n- Fix: rerun snapshot with short name.\n\n## CLOUDFLARE_DNS_AUDIT_TXT_SUBSTRING_GAP_20260701\n- Context: prior Cloudflare DNS audit checked web A/AAAA/CNAME state for root/www records.\n- Issue: it did not deeply inspect TXT/SPF contents for old provider IP substrings.\n- Evidence: user screenshot showed SPF TXT containing old 87.236.18.* value.\n- Impact: web routing is OK, but mail-related DNS may still contain stale provider data.\n- Rule: perform full DNS record audit including TXT/SPF/DKIM/DMARC/MX/CDN and substring search for old IPs before DNS cleanup.\n\n## CLOUDFLARE_DNS_CLEANUP_DRYRUN_QUOTE_563_20260701\n- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt.\n- Impact: proof 563 is invalid and no DNS cleanup was performed by it.\n- Rule: use copied scripts for Cloudflare DNS mutation instead of fragile inline nested quotes.\n\n## SPF_DUPLICATE_AFTER_565_20260701\n- Context: SPF cleanup command 565 attempted to replace stale SPF records.\n- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained.\n- Impact: domains temporarily had duplicate SPF records, which is invalid for mail validation.\n- Fix: delete all SPF TXT records for the five forum zones, then create exactly one v=spf1 -all TXT per zone.\n\n## MSMTP_SECRET_SOURCE_PARSE_ERROR_576_20260701\n- Context: proof 576 installed msmtp but sendmail auth test failed.\n- Issue: [SENSITIVE_PATH] was sourced as shell, but SMTP password contained shell-special characters; source failed and auth used an invalid/truncated secret path.\n- Impact: msmtp package installed, but mail sending was not proven working.\n- Rule: never source SMTP secret files containing arbitrary passwords; store password base64 and use msmtp passwordeval helper accessible to www-data.\n\n## FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701\n- Context: attempted to fix msmtp config with passwordeval helper.\n- Issue: one or more sendmail/PHP mail tests still failed.\n- Impact: XenForo mail sending is not yet proven.\n- Next step: verify SMTP credential/password and provider policy for noreply@pvepro.ru.\n\n## SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701\n- Context: SMTP password was exposed in terminal output during failed msmtp setup.\n- Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line.\n- Impact: treat that SMTP password as compromised.\n- Required action: rotate the mailbox/app password in the mail provider panel before any further SMTP testing.\n- Rule: never store arbitrary passwords as shell-sourceable KEY=value files; use non-printing secret storage only.\n\n## NETBIRD_MAILCOW_REPORTED_DOWN_AFTER_SMTP_INCIDENT_20260701\n- Context: user reported NetBird and Mailcow became unavailable after forum mail-transport work.\n- Impact: treat as active incident until service reachability and container/VM state are proven.\n- Rule: do not change DNS, firewall, proxy, mail, NetBird, or Mailcow configs before read-only triage proof.\n\n## FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701\n- Context: forum-prod SMTP/msmtp test used invalid or compromised credentials and triggered Mailcow netfilter warnings/ban for 95.84.154.183.\n- Impact: forum mail transport is not proven and must remain disabled until SMTP password is rotated and tested once carefully.\n- Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth.\n- Rule: do not test SMTP auth again until a fresh rotated mailbox/app password is available.\n\n## FORUM_OLD_CODEVIPE_CLOUD_BACKUP_STALE_AFTER_FRESH5_20260701\n- Context: pve02 codevipe-cloud-backup.timer existed before fresh five-forum rebuild.\n- Issue: old script points to /root/backups/codevipe and old remote path app/codevipe/forum-prod/local.\n- Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory contains old codevipe-public/codevipe.sql files, not current five-forum archive.\n- Impact: old timer must not be treated as valid current backup for all five forums.\n- Fix: create a new forum-prod fresh5 cloud backup using existing pve02 Mail-cloud rclone crypt remotes.\n\n## FORUM_RESTORE_DRILL_610_STATUS_FLAG_BUG_20260701\n- Context: restore drill proof 610 downloaded Mail.ru Cloud backup, verified split part SHA256, reconstructed archive, matched full archive SHA256, and confirmed all five forum DB/file archives in tar.\n- Issue: final RESTORE_DRILL_OK was set to no because the script tested an unassigned shell variable RCLONE_REMOTE_PVE02_MAIL_01_RC.\n- Impact: proof 610 final status flag is invalid, but its detailed integrity checks are valid.\n- Fix: rerun corrected restore validation against the downloaded cloud backup artifacts and produce proof 612.\n\n## FORUM_XENFORO_MAIL_SMOKE_TEST_FAILED_20260701\n- XenForo-level mail smoke test did not return success for all five forums.\n- Check proof 623 and msmtp log before retrying.\n\n## XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701\n- Context: custom XenForo mail smoke test 623 failed with Could not open input file /root/scripts/623_xenforo_mail_smoke.php.\n- Cause: the script was run as www-data from /root/scripts; /root is not traversable by www-data.\n- Impact: proof 623 is invalid and should not be used to judge mail delivery.\n- Superseding evidence: user ran XenForo built-in outgoing email test; message was delivered to Yandex but placed in Spam.\n- Next step: investigate deliverability/authentication headers and pvepro.ru SPF/DKIM/DMARC/PTR.\n\n## FORUM_NEXT_CHAT_KNOWN_ERRORS_AND_CLOSED_INCIDENTS_20260701\n\n### Closed / classified incidents\n- SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701:\n - A previous bad command sourced a raw SMTP secret and printed it.\n - Treat old password as compromised.\n - Later persistent SMTP was rebuilt using safe files and verified.\n - Never print or package secrets.\n\n- XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701:\n - Custom mail proof 623 failed with \"Could not open input file\".\n - Cause: PHP was run as www-data from /root/scripts, but /root is not traversable by www-data.\n - Impact: proof 623 is invalid and must not be used to judge mail delivery.\n - Superseded by user-observed built-in XenForo test and Yandex header proof.\n\n- FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701 / FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701:\n - Earlier SMTP attempts failed and triggered mailcow netfilter ban for forum public IP.\n - Ban was removed.\n - Persistent msmtp transport was later enabled and verified.\n - Mailcow and NetBird remained reachable after final tests.\n\n- SPF_DUPLICATE_AFTER_565_20260701:\n - Earlier SPF cleanup created duplicate SPF records.\n - Fixed by deleting duplicates and recreating exactly one SPF per forum domain.\n - Final forum domains use v=spf1 -all because sending uses noreply@pvepro.ru.\n\n- FORUM_RESTORE_DRILL_610_STATUS_FLAG_BUG_20260701:\n - Restore drill 610 had a status flag bug despite successful detailed checks.\n - Corrected restore drill 612 passed.\n\n- FORUM_DBTECH_TOR_TIMEOUT_CLASSIFIED_20260701:\n - One historical timeout row per forum to check.torproject.org.\n - DBTech/Security active, but dbtech_security_tornodes=0.\n - Classified as external timeout noise, not runtime failure.\n\n### Current non-blocking items\n- Yandex placed built-in XenForo test mail in Spam despite SPF pass and DKIM pass.\n- Classification: deliverability/reputation/content filtering, not server failure.\n- Recommended manual action: click “Не спам!” and add/check pvepro.ru in Yandex Postmaster.\n\n### Safety rules for next chat\n- Do not print secrets.\n- Do not download or upload:\n - [SENSITIVE_PATH]\n - /etc/msmtprc\n - /etc/msmtp/*\n - rclone configs\n - Cloudflare tokens\n - DB dumps\n - VM disks\n - backup archives\n- For handoff, only share the two truth files and selected non-secret proof files.\n\n## PARKED_DOMAINS_STAGE4_DNS01_PREFLIGHT_FAILED_20260701\n- Context: parked-domain public apply proof 634.\n- Issue: DNS-01 based apply was attempted before a clear CAN_APPLY=yes preflight result was visible in terminal output.\n- Observed: stage4 stopped safely with preflight_failed; CERTBOT_PRESENT=no, CERTBOT_DNS_CLOUDFLARE_PLUGIN=no, and CF_ZONE_ACCESS=no for newfi.ru, hapusya.ru and kingofwolk.ru.\n- Impact: no parked-domain NPMplus route/certificate changes were applied by stage4; public HTTPS validation still failed certificate verification.\n- Rule: do not use Cloudflare DNS-01 for these parked domains unless a token with explicit zone access is installed; because DNS already points to edge, prefer a copied-script HTTP-01 flow with temporary ACME challenge routes and content-specific validation.\n\n## PARKED_DOMAINS_STAGE5_BASH_LOCAL_SETU_BUG_20260701\n- Context: parked-domain HTTP-01 apply proof 635.\n- Issue: edge script used Bash `local id=\"$1\" ... conf=\"$WORK/.../$id.conf\"` and `local host=\"$1\" ... tmp=\"$WORK/.../$host.html\"` under `set -u`; dependent variables are not safe inside the same local assignment command.\n- Observed: `id: unbound variable`, `source can not be empty`, then `host: unbound variable`; certbot did not run, final HTTPS routes were not installed.\n- Impact: local parked page remained OK; public HTTPS remained not closed.\n- Rule: split dependent Bash local assignments into separate statements, syntax-check copied scripts before execution, and keep content-specific validation for parked routes.\n\n## PARKED_DOMAINS_STAGE6_TEMP_HTTP_ROOT_VALIDATION_BUG_20260701\n- Context: parked-domain HTTP-01 fixed apply proof 636.\n- Issue: temporary HTTP-01 route validation checked HTTP `/` for parked marker, but expected behavior before certificates is root HTTP 301 to HTTPS.\n- Observed: LOCAL_HTTP_VALIDATE returned HTTP_CODE=301 MARKER_OK=no, script rolled back its temporary route configs and did not run certbot.\n- Impact: local parked page remained OK; public HTTPS remained not closed.\n- Rule: validate `/.well-known/acme-challenge/` for HTTP-01 readiness, not HTTP `/`; also scan all active NPMplus nginx config dirs for unmanaged domain conflicts before writing routes.\n\n## PARKED_DOMAINS_STAGE7_NEWFI_ROOT_ACME_REDIRECT_20260701\n- Context: parked domains HTTP-01 Stage7 stopped before certbot because ACME validation path returned 301 for root newfi.ru while www.newfi.ru, hapusya.ru, www.hapusya.ru, kingofwolk.ru and www.kingofwolk.ru returned 200.\n- Impact: no final parked-domain HTTPS routes/certificates were applied by Stage7; temporary route configs were rolled back.\n- Rule: do not run another apply stage until active NPMplus nginx route selection for newfi.ru is diagnosed with nginx -T and Host-header probes.\n\n## PARKED_DOMAINS_STAGE8_DEFAULT_HTTP_TO_HTTPS_REDIRECT_20260701\n- Context: parked-domain route autopsy proof 638.\n- Finding: no active NPMplus config contains newfi.ru/hapusya.ru/kingofwolk.ru, but unknown HTTP hosts and ACME paths return 301 to HTTPS from NPMplus default routing.\n- Impact: HTTP-01 cannot work through the default server. A dedicated temporary ACME server block must be proven before certbot; do not run another certbot apply without a successful ACME-path probe for all six hostnames.\n- Rule: after every parked-domain temp route attempt, rollback the route and prove rollback before continuing.\n\n## PARKED_DOMAINS_STAGE9_NEWFI_EXACT_PROBE_NEEDED_20260701\n- Context: parked domains HTTP-01 probe stage9 installed one combined temporary server block for all six hostnames.\n- Issue: five hostnames returned ACME probe 200, but root newfi.ru still returned 301.\n- Impact: do not continue certbot/apply until exact newfi.ru route behavior is isolated.\n- Rule: run a temporary exact newfi.ru probe block and roll it back before any HTTP-01 apply.\n\n## PARKED_DOMAINS_STAGE10_EXACT_NEWFI_PROBE_OK_20260701\n- Context: parked-domain Stage10 proof 640.\n- Result: exact server block for newfi.ru on NPMplus intercepted both root and ACME paths with HTTP 200 and custom X-Parked-Probe markers.\n- Stage9 combined server_name block passed 5/6 but failed bare newfi.ru, so final HTTP-01 cutover must use separate exact per-host server blocks.\n- Rule: do not use one combined parked-domain server_name block for these domains; use exact host blocks and content-specific validation.\n\n## PARKED_DOMAINS_STAGE11_LOW_ID_EXACT_ROUTE_FAILED_20260701\n- Context: parked-domain Stage11 proof 641.\n- Result: exact per-host route files 700-705 passed nginx -t but bare newfi.ru still hit HTTP->HTTPS redirect on ACME path.\n- Earlier Stage10 proof showed exact newfi.ru route works when placed at high id 998.\n- Hypothesis: file/include priority or NPMplus generated ordering makes low id 700 unsuitable for bare newfi.ru in this runtime.\n- Rule: do not retry parked-domain HTTP-01 with low id 700-705. First prove fixed high-id exact blocks, then apply with those exact ids.\n\n## PARKED_DOMAINS_SWITCH_TO_DNS01_AFTER_HTTP01_FAILURES_20260701\n- Context: parked domains newfi.ru, hapusya.ru and kingofwolk.ru.\n- HTTP-01 attempts hit NPMplus default HTTP-to-HTTPS redirect/include-order problems.\n- Stage12 cleanup confirmed temp files 980-985 absent and nginx reload OK.\n- Rule: do not continue HTTP-01 for these parked domains; use DNS-01 with a dedicated Cloudflare token that has Zone Read and DNS Edit for the three zones.\n\n## PARKED_DOMAINS_STAGE15_FALSE_LOCAL_VALIDATE_ROLLBACK_20260701\n- Context: Stage15 DNS-01 succeeded and public validation showed parked page for most hosts, but local edge curl --resolve validation returned CURL_RC=60 and triggered rollback.\n- Impact: do not rerun Stage15 as-is.\n- Rule: for parked domains finalization, use existing DNS-01 certificates and validate from pve01 public HTTPS with content marker; do not use the failed local --resolve TLS check as the closure gate.\n\n## DOMAIN_ACTIONS_STAGE18_PLACEHOLDER_AND_RENEWAL_PROOF_20260701\n- Context: after parked domains and portfolio audit, operator requested removing domain enumeration from placeholder and checking certificate renewal paths.\n- Rule: Stage18 must not change DNS records or NPMplus route files. It may update placeholder HTML, run existing forum renewal script once for runtime proof, and run certbot dry-run for parked certificates using existing DNS-01 hooks.\n\n## GRAM1_ROOT_WWW_PLACEHOLDER_CLOSE_STAGE19_20260701\n- Context: operator requested gram1.ru root/www to use the existing placeholder page.\n- Rule: only root gram1.ru and www.gram1.ru may be routed. Existing gram1.ru subdomain NPMplus routes must not be changed.\n- Rule: use DNS-01 with a dedicated gram1 Cloudflare token; do not use HTTP-01.\n- Stage18 parked dry-run note: kingofwolk.ru dry-run hit Let's Encrypt rateLimited/service-busy after active certificate issuance and public HTTPS were already OK. Treat as transient external dry-run failure, not route/cert closure blocker.\n\n## PVEPRO_EDGE_LANDING_STAGE21_20260701\n- Context: pvepro.ru root/www currently point to the Mailcow VPS and return Mailcow with TLS hostname mismatch.\n- Decision: because pve01 has no SSH key access to the external VPS, do not modify the VPS. Instead move only pvepro.ru and www.pvepro.ru A records to edge and serve an edge landing page.\n- Safety rule: do not change mail.pvepro.ru, nb.pvepro.ru, MX, SPF, DKIM, DMARC or forum SMTP configuration.\n- Use a dedicated Cloudflare token for pvepro.ru. Do not print token values.\n\n## PVEPRO_STAGE21_LANDING_SERVICE_FAILED_20260701\n- Context: Stage21 stopped safely before DNS/cert/NPMplus route because local pvepro landing service returned CURL_RC=7 on 127.0.0.1:18089.\n- Impact: DNS root/www pvepro.ru remained on 185.139.214.215 and no route/cert changes were applied.\n- Rule: Stage22 must fix and prove the local landing service first, then proceed to DNS-01 cert, NPMplus route and root/www DNS only after local marker PVEPRO_LANDING_OK is reachable.\n\n## PVEPRO_STAGE22_VALIDATION_AND_COPY_GUARD_20260701\n- Context: Stage22 succeeded on edge-side landing, cert, NPMplus route and Cloudflare A upserts, but wrapper had a non-critical scp wildcard failure due __pycache__ and public validation used cached DNS for root pvepro.ru.\n- Mistakes recorded: do not scp scripts/* after local py_compile because __pycache__ may be present; do not continue after critical scp failure in future apply stages; do not classify PVEPro landing as NetBird just because it contains a link text; use Cloudflare authoritative A records and --resolve validation when DNS propagation may lag.\n- Stage23 must be read-only for infrastructure and may close only if edge route, Cloudflare A records, HTTPS landing marker, and mail/nb unchanged checks pass.\n\n## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701\n- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router.\n- Current limitation: there is no safe private remote access path to the dacha router yet.\n- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet.\n- Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists.\n\n## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701\n- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router.\n- Current limitation: there is no safe private remote access path to the dacha router yet.\n- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet.\n- Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists.\n\n## TAFTAUTO_WG_STAGE24_SECRET_AND_IMPORT_RULE_20260701\n- Context: configuring private management path for taftauto.ru dacha router.\n- Rule: do not print WireGuard private keys, PSK, or client config body in proof/chat.\n- Rule: generated client config may be stored only as root/debian mode 600 file and must be imported into the Netcraze router UI manually.\n- Rule: do not open router public admin or certificate deployment until WireGuard handshake is proven.\n\n## TAFTAUTO_WG_PSK_ROTATION_SCRIPT_BROKE_DATAPLANE_20260702\n- Assistant gave unsafe PSK rotation flow; after profile reimport dacha interface changed from Wireguard0 to Wireguard1 and data-plane broke until security-level, ACL and routes were restored on Wireguard1.\n- Proof 666 is superseded because it showed PUBLIC_SSH_22_STILL_OPEN.\n- Final fixed proof: /root/evidence/667_TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702_PROOF.txt\n\n## TAFTAUTO_CERTBOT_CLOUDFLARE_PLUGIN_MISSING_20260702\n- Certbot on edge rejected --dns-cloudflare-* arguments: dns-cloudflare plugin missing/not loaded.\n- No taftauto certificate issued in failed run; no DNS A records or NPMplus routes changed.\n\n## HOMEPAGE_SERVICES_YAML_BAD_INDENT_20260702\n- Assistant inserted Homepage services.yaml entries with wrong indentation; YAMLException bad indentation at line 227.\n- Fix: rebuild SmartApe under External Homelabs with 2-space item indentation, remove Router Moscow, rebuild Sites category.\n\n## HOMEPAGE_SERVICES_YAML_BAD_INDENT_REPAIR_20260702\n- Previous apply broke services.yaml indentation and did not follow YAML-aware rule.\n- Repair uses latest valid services.yaml backup, PyYAML parse/dump, validates before restart.\n\n## TAFTAUTO_CERTBOT_DRYRUN_BAD_FLAG_20260702\n- Assistant used unsupported certbot flag --manual-public-ip-logging-ok with certbot 4.0.0 during taftauto renew dry-run.\n- No certificate, DNS, NPMplus route, or deploy state changed by the failed dry-run.\n\n## TAFTAUTO_CERT_RENEW_DRYRUN_RATE_LIMITED_20260702\n- Proof 672 is superseded/partial: it printed STATUS=OK even though certbot dry-run failed.\n- Failure reason: Let's Encrypt staging/service returned rateLimited / Service busy; retry later.\n- Cloudflare manual auth and cleanup hooks did run successfully.\n- Deploy hook was installed and manually invoked successfully before the dry-run.\n- Do not retry immediately.\n\n## HOMEPAGE_XENFORO_INFO_INLINE_PYTHON_SYNTAX_ERROR_20260702\n- Assistant used invalid inline Python syntax with def after semicolon while adding XenForo.info to Homepage External.\n- Failed before services.yaml write.\n\n## HOMEPAGE_USEFUL_ROUTER_NAME_MISMATCH_20260702\n- Previous apply looked for Russian Router card name, but active Homepage Useful group contains Router and Public Domain.\n- Failed before write. Fix uses YAML-aware edit with exact names Router/Public Domain.\n\n## HOMEPAGE_USEFUL_ROUTER_CARD_NOT_FOUND_20260702\n- Apply expected Useful -> Router in active services.yaml, but YAML parse did not find it. No write was performed.\n- Need verify active Homepage config path, docker mount, and any alternate services.yaml before next apply.\n\n## 20260702_CF_TOKEN_AUDIT_BROKEN_COMMAND\n- Ошибка: ассистент дал хрупкую команду аудита Cloudflare-токенов с вложенным heredoc/Python через ssh.\n- Факт: bash начал интерпретировать Python-код, появились syntax error и вторичный prompt >.\n- Правило: не давать nested heredoc/сложные кавычки через ssh; для длинной логики сначала класть скрипт отдельным файлом, затем запускать его.\n- Статус: аудит токенов не выполнен, инфраструктурные изменения не должны считаться применёнными.\n\n## LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE\n- Do not treat e3qxxx.netbird.selfhosted / 100.100.125.70 and e3qxxx-183-106.netbird.selfhosted / 100.100.183.106 as VPS egress nodes; they are user mobile peers.\n- Correct egress candidates are relay.pvepro.ru / relay.netbird.selfhosted / 100.100.19.1 for Moldova and mail.pvepro.ru / mail.netbird.selfhosted / 100.100.147.204 for USA.\n- Access Policy edge-vm -> relay/mail only proves peer reachability. It is not an internet exit route.\n- Required NetBird fix for CrowdSec CAPI was Network Routing exit route 0.0.0.0/0 distributed to edge-vm group, routing peer relay, masquerade/NAT enabled.\n- Before attempting CrowdSec CAPI, verify on edge-vm: ip route get 1.1.1.1 uses wt0, wg allowed-ips contains 0.0.0.0/0, and Cloudflare trace no longer shows home IP 95.84.154.183.\n\n## LESSON_20260702_HOMEPAGE_MONITORS_REPAIR_NOT_DELETE\n- Do not delete or disable Homepage siteMonitor fields to hide red badges.\n- Fix the underlying monitor path, DNS, proxy route, or local health-wrapper instead.\n- Do not touch Cloudflare Homepage card when operator says it is green and opens correctly.\n- Before changing Homepage, identify the live config host; in this state live Homepage is on core-apps [PRIVATE_IP], not edge-vm.\n\n## LESSON_20260702_KUMA_ADD_ONE_MONITOR_ONLY\n- When asked to fix one missing Uptime Kuma service, add only one monitor, then generate proposals separately.\n- Before direct Kuma DB mutation, stop the container and create a DB backup.\n- Verify DB integrity before starting Kuma again.\n- Do not touch Cloudflare when operator says it is green and opens correctly.\n\n## LESSON_20260702_DOCKGE_STALE_STACKS_NOT_CONTAINERS\n- Dockge inactive items after migration can be stale compose folders, not stopped containers.\n- First classify runtime projects across all Docker hosts before deleting or archiving anything.\n- For remote Dockge visibility, projects must be under /opt/stacks or bind-mounted there.\n- Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions.\n\n## STAGE4B_MASKED_SQL_QUERY_FAILURE_20260714\n- Previous Stage4B read-only audit returned empty observation-summary, metadata-key, trigger and function blocks because nested SSH/SQL quoting broke queries while stderr was hidden.\n- Empty blocks are query failures, not proof that metadata, triggers or functions are absent.\n- Future PostgreSQL audits must use ON_ERROR_STOP, visible stderr, independent query RC and quote-safe dollar literals.\n\n## STAGE4C_SCHEMA_MIGRATIONS_ID_ASSUMPTION_20260714\n- Stage4C read-only preflight incorrectly assumed schema_migrations has a column named id.\n- PostgreSQL returned column id does not exist and ON_ERROR_STOP correctly stopped all following SQL checks.\n- Production DB and application were not changed. Future migration audits must inspect information_schema first and query migration rows without assuming column names.\n\n## STAGE4C_PGDUMP_DEV_NULL_FSYNC_INVALID_PROBE_20260714\n- Stage4C custom-format pg_dump capability probe incorrectly used /dev/null as the output file.\n- pg_dump failed only because fsync on /dev/null returned Invalid argument.\n- Production database and application were not changed.\n- Future custom dump capability checks must use a real temporary regular file, validate it with pg_restore --list, and remove it afterward.\n\n## STAGE4C_REMOTE_PREP_THIRD_SUBSTEP_FAILED_20260714\n- Isolated migration dry-run did not start: REMOTE_PREP_RC=0|0|1.\n- Remote directory creation and SCP succeeded; the third ownership/mode preparation substep returned RC=1.\n- Production database and application were not changed.\n- Exact cause requires read-only residual directory and permission inspection before cleanup or retry.\n\n## STAGE4C_REMOTE_PREP_GLOB_AFTER_CHMOD_ROOT_CAUSE_20260714\n- Exact cause of REMOTE_PREP_RC=0|0|1: after the directory became postgres-owned mode 0700, the unprivileged debian shell could not traverse it and could not expand the wildcard used by chmod.\n- Candidate files and SHA256 verification were valid; no temporary database was created and production was unchanged.\n- Future preparation must use exact privileged paths or a verified shared group, never an unprivileged wildcard after restrictive chmod.\n\n## STAGE4C_MIGRATION003_FINGERPRINT_DOLLAR_QUOTE_FAILURE_20260714\n- Migration 003 isolated dry-run failed at fingerprint CHECK because the regex end anchor collided with the SQL dollar-quote delimiter and produced an unterminated dollar-quoted string.\n- The migration transaction rolled back, the temporary database was removed, and production remained unchanged.\n- Fingerprint validation must use length plus translate without a regex dollar anchor; corrected candidate must pass a fresh isolated dry-run before production consideration.\n\n## STAGE4C_SEAL_OUTER_RC_MASKING_20260714\n- Stage4C final seal correctly refused creation because overall health was WARN, returning internal RC=40.\n- The former wrapper then printed a second misleading COMMAND_RC=0 because top-level true masked the internal result.\n- Future commands must capture the guarded subshell through PIPESTATUS and print exactly one authoritative COMMAND_RC before the mandatory final true.\n- Production database, application and services were unchanged.\n\n## STAGE4C_DIAGNOSIS_EXPECTED_DERIVED_FILE_ON_VM180_20260714\n- Previous diagnosis incorrectly treated cluster-admin-restricted-probes.txt as a required source file on VM180.\n- The file is a central derived health artifact on pve01; VM180 supplies full-observer evidence instead.\n- This false expectation caused diagnostic RC=32 without any infrastructure failure or production change.\n\n## STAGE4C_NO_JOBS_RUNNING_TEXT_COUNTED_AS_JOB_20260714\n- Stage4C seal preflight incorrectly counted the systemctl informational text No jobs running as one active job.\n- The parser counted every nonempty output line instead of accepting only rows beginning with a numeric systemd Job ID.\n- No service was started and no production state changed during the blocked attempt.\n- Future job counts must match a numeric first field only.\n\n## STAGE4D_ADAPTER_SOURCE_AUDIT_ABORTED_BEFORE_FINAL_MARKERS_20260714\n- The evidence-only Stage4D adapter source audit returned RC=1 before emitting its VM180 completion and final audit markers.\n- The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults.\n- Production, database, application, services, timers, health and desired-state were unchanged.\n- Inspect the preserved runner stdout, stderr and result JSON before correcting or rerunning the task.\n\n## STAGE4D_LOCAL_EXPANSION_OF_REMOTE_Q_UNDER_NOUNSET_20260714\n- The first Stage4D adapter source audit embedded a large remote program inside a locally double-quoted SSH argument.\n- Local shell expansion reached the remote variable Q while nounset was enabled, causing Q: unbound variable before SSH execution.\n- VM180 and PostgreSQL audits did not start; production state was unchanged.\n- Remote shell and SQL programs must be separate syntax-checked files passed through SSH stdin.\n\n## STAGE4D_REMOTE_PYTHON311_ADAPTER_SELFTEST_RC1_20260714\n- Canonical adapter candidate passed local Python 3.13 self-tests with 48 envelopes but returned RC=1 on VM180 Python 3.11.2.\n- Remote upload and cleanup succeeded, and production database remained 0|0|OK.\n- The exact traceback and failing assertion must be captured by an isolated identical reproduction before changing the candidate.\n\n## STAGE4D_PYTHON311_FSTRING_COMPATIBILITY_AND_REPRO_VALIDATOR_20260714\n- Adapter candidate passed Python 3.13 but failed Python 3.11 parsing at Path(row[\"path\"]) inside a double-quoted f-string.\n- Exact fix is Python 3.11-compatible quoting: Path(row['path']).\n- The reproduction command captured the expected compile failure but incorrectly returned CAPTURE_BAD=1 because its validator required compile RC=0.\n- Production database remained 0|0|OK and desired-state remained clean.\n- Future adapter candidates must pass syntax and full self-tests on both pve01 and VM180 runtimes before acceptance.\n\n## STAGE4E_DESIGN_VALIDATOR_ABORTED_BEFORE_LOCAL_VALIDATION_MARKER_20260714\n- Restricted probe-agent design candidate compiled successfully but its local validator exited before producing LOCAL_VALIDATION_RC.\n- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults.\n- Active external probes were not executed and production state was unchanged.\n- Inspect the preserved validator traceback and exact assertion before modifying the candidate.\n\n## STAGE4E_VALIDATOR_SCANNED_PYC_AND_OWN_FORBIDDEN_LITERALS_20260714\n- Stage4E design validator recursively read every candidate file as UTF-8 after py_compile had created binary __pycache__ bytecode.\n- This caused UnicodeDecodeError before any design assertion failed.\n- The phase-wide forbidden-pattern scan also included validate_design.py itself, which necessarily contains the prohibited literals it verifies.\n- Validators must use an explicit text-file allowlist and scans must exclude validator implementation and binary artifacts.\n- Production, database, services, timers and desired-state were unchanged; active external probes were not executed.\n\n## STAGE4E_V2_VALIDATOR_FAILED_STATIC_COMPILE_20260714\n- Stage4E design candidate v2 failed static compilation before local design validation started.\n- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects.\n- The generated validator must be inspected at the exact SyntaxError line before another candidate is created.\n- Production, database, application, services, timers, health and desired-state were unchanged; active external probes were not executed.\n\n## STAGE4E_PIPESTATUS_LOST_AFTER_FIRST_ASSIGNMENT_20260714\n- VM180 validation copy used a pipeline and read PIPESTATUS in two separate assignments.\n- The first assignment reset PIPESTATUS, leaving COPY_RC empty and preventing remote validation and cleanup.\n- Retry must use explicit SCP operations without pipeline status parsing.\n- Production remained unchanged and active external probes were not executed.\n\n## STAGE4F_VALIDATOR_EXPECTED_HEALTH_REFRESH_FROM_NONWRITING_BACKUP_SCRIPT_20260714\n- The controlled backup service completed with Result=success and ExecMainStatus=0.\n- Validation incorrectly required cluster-admin-incident-engine-backup.txt to receive a fresh timestamp.\n- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.\n- Stage4F backup proof must use the service result, fresh latest JSON, produced artifacts and off-host/restore evidence instead.\n- The backup must not be rerun merely to satisfy the invalid health-file freshness assertion.\n\n## STRICT_RULE_20260714_CLOSE_TAILS_IMMEDIATELY\n- Failure class: переход к следующей задаче при наличии незакрытого хвоста.\n- Mandatory anti-regression: после ошибки запрещено продолжать другой scope до точной root cause, исправления, повторной проверки, очистки, proof и seal.\n- Closure gate: UNRESOLVED_TAIL_COUNT=0; BLOCKER_COUNT=0; TEMPORARY_ARTIFACT_COUNT=0; ROLLBACK_VERIFIED=YES; HEALTH=OK; PROOF_UPDATED=YES; REFERENCE_UPDATED=YES.\n- Если закрытие невозможно из-за внешней зависимости, опасной неоднозначности или обязательного решения пользователя, фиксировать BLOCKED/OPEN и не заявлять CLOSED.\n- Proof: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/report.txt\n\n## ERROR_20260714_DEPENDENCY_FACT_QUERY_GLOBAL_LIMIT\n- Symptom: dependency audit вернул только latest_collector_status.\n- Root cause: ORDER BY 1 LIMIT 1 в конце UNION ALL применился ко всему набору фактов.\n- Correction: latest collector status извлекается скалярным подзапросом внутри CTE facts; глобальный LIMIT отсутствует.\n- Anti-regression: multi-fact SQL audit обязан проверять точное ожидаемое число строк до извлечения значений.\n- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt\n- Status: CLOSED.\n\n## ERROR_20260714_COLLECTOR_PATCH_REQUIREMENT_PATTERN_MISMATCH\n- Symptom: MIGRATION_PLAN_COLLECTOR_PATCH_REQUIRED_COUNT=1 при ожидаемых двух подтверждениях.\n- Root cause: узкий шаблон учитывал collector_patch_required, но не collector_patch_required_after_stage4c.\n- Correction: оба поля проверяются отдельно, затем складываются как semantic requirement count.\n- Anti-regression: связанные JSON-контракты проверяются по собственным точным именам полей, а не одним унифицированным grep-шаблоном.\n- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt\n- Status: CLOSED; PRIOR_TAIL_UNRESOLVED_COUNT=0.\n\n## ERROR_20260714_PRIVILEGED_FILE_REDIRECTION_BEFORE_SUDO\n- Symptom: bash reported Permission denied while counting collector.py lines.\n- Root cause: `< collector.py` was opened by the unprivileged caller shell before sudo executed wc.\n- Correction: run sudo wc -l collector.py without caller-side input redirection.\n- Anti-regression: never combine sudo with `< protected-file`; pass the protected path as an argument to the privileged command.\n- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt\n- Status: CLOSED.\n\n## ERROR_20260714_COLLECTOR_RUN_COLUMN_ASSUMPTION\n- Symptom: SQL failed because completed_at did not exist.\n- Root cause: collector_runs columns were assumed instead of read from information_schema; actual names are finished_at and error_text.\n- Correction: assert required and forbidden column counts before querying recent runs.\n- Anti-regression: every schema-sensitive audit must discover and validate exact column names before the main query.\n- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n\n## ERROR_20260714_REMOTE_STDIN_ARGUMENT_SHIFT\n- Symptom: remote harness выполнил chmod для пути bash.\n- Root cause: дополнительный positional token bash был передан после SSH command и стал первым аргументом stdin-скрипта.\n- Correction: remote command вызывает bash -s -- с ровно тремя явно встроенными аргументами: candidate path, SHA256 и byte count.\n- Anti-regression: stdin-скрипт обязан валидировать $1 по разрешённому path pattern и затем подтверждать точные SHA256 и byte count; запрещено добавлять отдельный bash-token после remote command.\n- Closure proof: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/report.txt\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n\n## ERROR_20260714_OVERSIZED_COMMAND_PARSE_FAILURE\n- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`.\n- Root cause: чрезмерно длинная интерактивная команда была обрезана или повреждена при передаче до полного разбора Bash.\n- Impact: side-effect audit подтвердил production state 0|0|OK, неизменный collector, активный timer и отсутствие временных БД, каталогов и процессов.\n- Correction: монолитные команды запрещены; сложные действия выполняются через отдельный проверяемый task-скрипт и короткий launcher.\n- Anti-regression: interactive payload target <= 8000 bytes; syntax-check перед запуском; remote output сохраняется независимо от RC.\n- Closure proof: /var/lib/homelab-change-sets/operator-command-size-guard-v1/20260714T182653Z/report.txt\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n\n## ERROR_20260714_HOMELAB_ADMIN_HELP_RC_ASSUMPTION\n- Symptom: template audit завершился HOMELAB_ADMIN_HELP_FAILED.\n- Root cause: был ошибочно ожидаем RC=0 от неподдерживаемого аргумента --help.\n- Actual contract: неизвестный аргумент печатает usage и возвращает RC=64.\n- Correction: проверять version либо считать точную usage-строку вместе с RC=64 штатным результатом.\n- Anti-regression: перед проверкой CLI сначала фиксировать поддерживаемые команды; не предполагать GNU-style --help.\n- Closure proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v1/20260714T183338Z/report.txt\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n\n## ERROR_20260715_EXPECTED_NEGATIVE_RC_TRAP_CLASSIFICATION\n- Symptom: schema-v2 task v3 build stopped while intentionally testing the --invalid argument path.\n- Root cause: an expected RC64 was executed while the generic ERR trap remained active.\n- Correction: expected failures are evaluated only through an explicit if/else branch with immediate RC capture.\nANTI_REGRESSION_TOKEN=EXPECTED_RC64_INTERCEPTED_ERR_TRAP\n- Required pattern: if command; then rc=0; else rc=$?; fi, followed by a separate exact expected-RC assertion.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n\n## ERROR_20260715_DUPLICATE_MACHINE_MARKER_COUNT\n- Symptom: error-register candidate construction stopped with RC1 before applying the candidate.\n- Root cause: the same marker text appeared in both a heading and a descriptive line, while the guard expected one substring occurrence.\n- Correction: headings and prose do not contain machine-token values; validation counts only complete exact token lines.\nANTI_REGRESSION_TOKEN=DUPLICATE_MACHINE_TOKEN_IN_HEADING_AND_BODY\n- Required pattern: grep -Fxc against a complete machine line, never grep -Foc against an unrestricted substring.\n- Production impact: none.\n- Temporary artifacts: removed and verified.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T02:56:19Z\n\n## ERROR_20260715_OUTER_WRAPPER_SHELL_SYNTAX\n- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis.\n- Root cause: the outer wrapper contained an unbalanced compound shell construct before the closing subshell delimiter.\n- Impact: parsing failed before the start marker, guards and homelab-admin invocation; no task phase or infrastructure action executed.\n- Correction: use a linear wrapper with explicitly closed if/case blocks and avoid nested brace groups in conditional expressions.\nANTI_REGRESSION_TOKEN=OUTER_WRAPPER_UNBALANCED_SUBSHELL_SYNTAX\n- Anti-regression: after any parser-level failure, treat the attempted phase as not executed and verify package and run-state before retrying.\n- Production impact: none.\n- Task package impact: none.\n- Temporary artifacts: none.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T03:24:05Z\n\n## ERROR_20260715_INLINE_REMOTE_SHELL_IN_APPLY_PHASE\n- Symptom: homelab-admin lint rejected Stage4H task v6 with ERR-002-INLINE-REMOTE-SHELL.\n- Root cause: apply.sh invoked bash -s through an inline quoted SSH program and streamed the remote script through stdin.\n- Correction: copy the declared remote script to an isolated fixed path on VM180 and invoke that file directly with positional arguments.\nANTI_REGRESSION_TOKEN=INLINE_REMOTE_SHELL_REPLACED_BY_STAGED_REMOTE_SCRIPT_PATH\n- Anti-regression: phase scripts may perform transport orchestration, but remote shell program bodies must exist as separately declared, syntax-checked remote script files.\n- Remote stdout and stderr must be preserved before evaluating the remote return code.\n- Task v6 status: REJECTED_BY_LINT; never executed and never mutated.\n- Production database impact: none.\n- Live collector impact: none.\n- Temporary database and remote root count after rejection: zero.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T04:13:46Z\n\n## ERROR_20260715_OPTIONAL_RUN_DIRECTORY_FIND_UNDER_PIPEFAIL\n- Symptom: task-v7 builder stopped while counting formal runs for rejected task v6.\n- Root cause: find was executed against an optional directory that did not exist; under pipefail the assignment returned RC1.\n- Correction: test directory existence first and assign zero without invoking find when it is absent.\nANTI_REGRESSION_TOKEN=MISSING_OPTIONAL_RUN_DIRECTORY_TREATED_AS_ZERO\n- Anti-regression: optional paths must have an explicit existence branch before find under pipefail.\n- Production impact: none.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T04:44:06Z\n\n## ERROR_20260715_PG_CONSTRAINT_CONTYPE_CHAR_CONCAT\n- Symptom: isolated Stage4H acceptance failed during schema baseline capture.\n- Root cause: pg_constraint.contype uses the internal PostgreSQL char type and was concatenated without an explicit text cast.\n- Correction: use contype::text or CAST(contype AS text).\nANTI_REGRESSION_TOKEN=PG_CATALOG_CHAR_CONCAT_REQUIRES_EXPLICIT_TEXT_CAST\n- Machine rule: ERR-014-PG-CATALOG-CHAR-CONCAT.\n- Negative self-test: uncast expression rejected with RC64.\n- Positive self-test: explicit text cast accepted with RC0.\n- Task v7 mutated: no.\n- Production impact: none.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T06:27:06Z\n\n## ERROR_20260715_LIVE_COLLECTOR_VERIFIER_WRONG_PATH\n- Symptom: verified linter installation was rolled back after the final collector integrity guard returned UNKNOWN.\n- Root cause: verifier queried a nonexistent path instead of the canonical live collector path declared by the task acceptance script.\n- Canonical path: /opt/cluster-admin-incident-engine/collector.py.\n- Incorrect path: /opt/homelab-cluster-admin/cluster-admin-collector.py.\nANTI_REGRESSION_TOKEN=LIVE_COLLECTOR_VERIFIER_PATH_MUST_MATCH_TASK_CANONICAL_PATH\n- Correction: derive and guard the live collector path from the immutable task contract before executing the hash query.\n- Production impact: none; the canonical collector hash remained unchanged.\n- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\n- Registered at: 2026-07-15T06:27:06Z\n\nAUTOREFRESH_INLINE_FAILURE_20260721\nAttempts 048 and 049 did not appear in immutable history.\nRule: use validated local helper scripts and a simple scheduler entry; never embed the full collector in cron or nested SSH quoting.\n\nAUTOREFRESH_WRAPPER_PUBLICATION_FAILURE_20260721\nCommands 044-051 did not reach immutable history through homelab-chat-run.\nResolution: hourly snapshots use direct scp plus homelab-runtime-receive publication.\n\nAUTOREFRESH_052_NOT_PUBLISHED_20260721\nCONTEXT-AUTOREFRESH-ACTIVATE-052 did not appear in immutable history; direct publisher retry requires sanitization before transmission.\n\nIMMUTABLE_HISTORY_FALSE_NEGATIVE_20260721\nBACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative.\nRule: verify the exact COMMAND_ID result, wrapper publication status and commit SHA; never infer absence from a cached directory listing.\n\nBACKUP_UNSAFE_SOURCE_CAPTURE_071_PARSE_ERROR_20260721\nBACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.\nResolution: avoid nested quote layers; parse targets first and pass file paths as simple SSH arguments.\nDETAIL=SECTION_ERROR_REGISTER_END\nDETAIL=SECTION_STRUCTURED_ERRORS_INDEX_BEGIN\n{\n \"schema_version\": 1,\n \"channel\": \"homelab-runtime\",\n \"command_id\": \"ERRORS-INDEX-004\",\n \"status\": \"OK\",\n \"rc\": 0,\n \"host\": \"pve01\",\n \"mode\": \"read-only\",\n \"component\": \"error-ledger\",\n \"started_at_utc\": \"2026-07-21T07:22:22Z\",\n \"finished_at_utc\": \"2026-07-21T07:22:22Z\",\n \"reference_register_checked\": true,\n \"reference_sha256\": \"f6f7749fda946015116cdbbd754c7e7f20356b4a6cf4df330a1b0b8fdf59cbb2\",\n \"error_register_checked\": true,\n \"error_register_sha256\": \"ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc\",\n \"command_sha256\": \"a45b24ab6a161508f31ee8a6b954c045e74824b4d12adfef6f298cfac6111016\",\n \"duplicate_failed_command_blocked\": false,\n \"block_reason\": null,\n \"execution_started\": true,\n \"changes_made\": false,\n \"sanitized\": true,\n \"secrets_included\": false,\n \"private_addresses_included\": false,\n \"raw_evidence_retained_locally\": true,\n \"raw_evidence_sha256\": \"655ec287a9095e799677cb7f87b08d4119cbdf9027d7097fa610da5a022ab30b\",\n \"sanitized_output_sha256\": \"655ec287a9095e799677cb7f87b08d4119cbdf9027d7097fa610da5a022ab30b\",\n \"output_truncated_in_json\": false,\n \"full_sanitized_output_url\": \"https://git.gram1.ru/.well-known/homelab-runtime/latest.txt\",\n \"output\": \"{\\n \\\"schema_version\\\": 1,\\n \\\"status\\\": \\\"READY\\\",\\n \\\"generated_at_utc\\\": \\\"2026-07-21T07:22:22.821762Z\\\",\\n \\\"source\\\": {\\n \\\"path\\\": \\\"/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\\\",\\n \\\"sha256\\\": \\\"ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc\\\",\\n \\\"line_count\\\": 934,\\n \\\"sanitized\\\": true\\n },\\n \\\"summary\\\": {\\n \\\"entry_count\\\": 154,\\n \\\"rule_count\\\": 90,\\n \\\"duplicate_entry_ids\\\": [],\\n \\\"duplicate_entry_signatures\\\": [\\n \\\"9cd06e064f5a86be9626c4f9a2a819b8d0e71b03309c705fe0e34e1e59f81b73\\\"\\n ],\\n \\\"duplicate_rule_signatures\\\": [\\n \\\"b936ae30ffb935118585b5a133d9c32643946a3ba5034b85f42bae000518d49a\\\"\\n ]\\n },\\n \\\"entries\\\": [\\n {\\n \\\"id\\\": \\\"ERR-H-L1-HOMELAB-ASSISTANT-ERROR-REGISTER\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 1,\\n \\\"source_line\\\": 1,\\n \\\"title\\\": \\\"HOMELAB ASSISTANT ERROR REGISTER\\\",\\n \\\"summary\\\": \\\"Назначение: перед каждой следующей командой сверяться с этим файлом.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"cd4b4d42c26702b03e3e81a620c46dbe3d498f3f2233215024a5c2a49cc1dbfc\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-1-L6\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 6,\\n \\\"title\\\": \\\"Повторно дал слишком большой интерактивный paste в shell.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"f732778534f22b73e31dead215c9c7078f48a552a5341ae3830eea618885e8a7\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-2-L7\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 7,\\n \\\"title\\\": \\\"Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"b13c8946e274927bd041960c76208b6f62e22d46f07233388878c1f47afc426a\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-3-L8\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 8,\\n \\\"title\\\": \\\"Дал генератор справочника прямо в терминал вместо безопасного маленького шага.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"8de70b8d263233557a33a065ee49004ef01e328de0d736eb9e328930119458be\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-4-L9\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 9,\\n \\\"title\\\": \\\"Нарушил своё же правило: не давать длинные вложенные команды с кавычками.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"d1bf3e42845e6c9d25511814c1da9b21cd8001474f17fd60240194f812f06fe0\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L5-КРИТИЧЕСКИЕ-ОШИБКИ-АССИСТЕНТА\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 5,\\n \\\"title\\\": \\\"Критические ошибки ассистента\\\",\\n \\\"summary\\\": \\\"1. Повторно дал слишком большой интерактивный paste в shell. 2. Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт. 3. Дал генератор справочника прямо в терминал вместо безопасного маленького шага. 4. Нарушил своё же правило: не давать длинные вложенные команды с кавычками.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"0ba065b54f515fe668143ec87d006ea8f27639e6529fe46367ac9acd6f2e28d6\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L11-ЖЁСТКИЕ-ПРАВИЛА-ПЕРЕД-КАЖДОЙ-КОМАНДОЙ\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 11,\\n \\\"title\\\": \\\"Жёсткие правила перед каждой командой\\\",\\n \\\"summary\\\": \\\"CHECK-1: команда не должна быть большим paste. CHECK-2: команда не должна содержать большой here-doc. CHECK-3: команда не должна смешивать Markdown, backticks и shell-логику. CHECK-4: команда не должна иметь вложенный ssh с несколькими уровнями кавычек. CHECK-5: команда не должна печатать секреты. CHECK-6: если создаётся файл, сначала маленький безопасный шаг, потом проверка. CHECK-7: если команда длиннее 8 строк, её нельзя давать в интерактивный shell. CHECK-8: для edge-vm использовать debian@[PRIVATE_IP] и sudo.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"7ac085a9df3e592d1d7d6686ca4980278110a07e296ddc0253cedb075cc08d84\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L23-ТЕКУЩИЕ-ВАЖНЫЕ-ФАКТЫ\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 23,\\n \\\"title\\\": \\\"Текущие важные факты\\\",\\n \\\"summary\\\": \\\"Internal network: [PRIVATE_IP]/24. Migration config: migration: secure,network=[PRIVATE_IP]/24. Corosync remains on [PRIVATE_IP]/12/13. VM160 forum-prod is not in Proxmox nightly backup. VM130 edge-vm has secondary disk backup=0 risk. 05_edge_compose_safe.tgz quarantined.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"ecd7f734df435c41c5f01c9fd7f9371a4012c6019493072741c45c6eaf3ed383\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-11-L35\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 35,\\n \\\"title\\\": \\\"Ошибка: считать offhost OK после failed rsync.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"aa8a2a5cc3228d73d2feadd69fa2fb1d0f76d020280f69921ac5c00dcfc1f9fd\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-12-L40\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 40,\\n \\\"title\\\": \\\"Ошибка: широкий secret-поиск по /opt/stacks дал шум.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"da33bb4eaab4ca4827c1e122c4292fa4181a7607db77ebce5d185b867bfe0ac3\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-13-L45\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 45,\\n \\\"title\\\": \\\"Ошибка: SQL с одинарными кавычками внутри одинарной SSH-команды ломается.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"888cbc312835f6303e74db4c5ed2dcaef1e552e657f83ca6f3f82834eb06c23e\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-14-L50\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 50,\\n \\\"title\\\": \\\"Ошибка: путать контекст входа и узел выполнения.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"97eec6b76b51ffbc78105ce4e450846c5ff1c85dfbcdfeb3e6c687addd6a1391\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-15-L57\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 57,\\n \\\"title\\\": \\\"Ошибка: повторно нарушено правило №13 после его добавления.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"95160c6fda4d8bc77fc542236363b182ba66a7314a8032379b033a0feba5e3b2\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-16-L63\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 63,\\n \\\"title\\\": \\\"Основополагающее правило: перед каждой командой ассистент обязан явно написать: Сверка с файлом ошибок: CHECK пройден.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"254db5ca3a2611269de7af81ececd43035134e01a8e629c34434913c49eeeb41\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-19-L68\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 68,\\n \\\"title\\\": \\\"Уточнение формата: не писать отдельную строку сверки перед каждой командой в пакете.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"a77bda86805176e2dc721961454b31a5bff15fe924bd43496475c5816a8f9650\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-20-L73\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 73,\\n \\\"title\\\": \\\"Ошибка: широкий XenForo SMTP-поиск по src дал шум исходников.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"bed26484549b8d47c75ee6150c206503c05074cc8a5b25dfde0735596aaf956c\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-21-L77\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 77,\\n \\\"title\\\": \\\"Ошибка: nested PHP php -r дал Parse error на forum-prod.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"1198d8f0576c5dc8e3883ac7d7e423c8391e946c73bf5ac3cf6063278a68a5af\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-22-L82\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 82,\\n \\\"title\\\": \\\"Ошибка: самодельный base64 PHP для SMTP auth сломан.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c629522366fb27522bec6dd9b9b24f532bbc9c3372fbf4d706673bf283bdf498\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-23-L87\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 87,\\n \\\"title\\\": \\\"Ошибка: MariaDB не поддержала cast('true' as json) в JSON_SET.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"4faccf6aa3103144040fd4efb62dd82e23d9f28ecfba72502293f6e2f060b819\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-24-L92\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 92,\\n \\\"title\\\": \\\"Ошибка: exit 1 в interactive-check закрыл SSH-сессию.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c522fbe5bc7aadfc904f1bde7328952b554884941afb3558e60cfe6a4ed378cd\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-25-L97\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 97,\\n \\\"title\\\": \\\"Ошибка анализа: не надо слепо включать smtpSsl=true при smtpPort=587.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"17417cdf64e208886d887fd449a612090a81aa5a8f47944f17993821ad5e8235\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-26-L102\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 102,\\n \\\"title\\\": \\\"Ошибка: openssl -crlf вместе с ручным CRLF ломает SMTP AUTH checker.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"3f83176fadcfeb2318a3e3fc75c13e3944d3fef3c0aa7aef40db3736b27e794a\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-27-L108\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 108,\\n \\\"title\\\": \\\"Ошибка: Python heredoc внутри ssh сломал not_ok диагностику.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"437f761644210f81c8ca3333410a58585c7b74c54a983b9d84159d98c49b4847\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-28-L112\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 112,\\n \\\"title\\\": \\\"Ошибка: docker compose ls --format json | wc -l дал ложный COMPOSE_PROJECT_COUNT=1.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"752e0913f85275527a448868cce24aa85cf06949ea82e066e4dc0fc2e857d1ad\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-29-L116\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 116,\\n \\\"title\\\": \\\"Ошибка: monitoring compact status искал неверные имена health-файлов.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"61df0580bfb53db159bb0c3fced23235437e823a877822e0b8aebb7518453d4d\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-30-L121\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 121,\\n \\\"title\\\": \\\"Ошибка/инцидент: Prometheus API HTTP_CODE=000 после monitoring reference.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c0dac19354e77153d3f61d53659a8e694c57308bc564e03c833cb68831e88031\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-31-L125\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 125,\\n \\\"title\\\": \\\"Строгое правило: перед каждой командой сверяться с файлом ошибок и справочником.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"fa4e49944e2caf8ba4f26a6064bfb465c035e8bc3184244c0314812e81bd6df5\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-32-L130\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 130,\\n \\\"title\\\": \\\"Ошибка: 153_PROMETHEUS_TARGETS_SETTLED_PROOF содержит SyntaxError из-за вложенного Python внутри SSH.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"13b3234027d04af4194a5352dda2a5bd21ab955751eb161b953637347da8febf\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-34-L135\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 135,\\n \\\"title\\\": \\\"Ошибка операционного ввода: полный терминальный лог MobaXterm попал в shell.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"9a7930e756982125ac45f47a4adf7dfdb10906bd950db4ddae3659021e01bc64\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-33-L140\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 140,\\n \\\"title\\\": \\\"Security finding: root authorized_keys на PVE-нодах имел права 777.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"44a7ef2e7d0428388c9b30d12ecfad460caaca457b5d283ebf00ae87c1b2119c\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-35-L144\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 144,\\n \\\"title\\\": \\\"Ошибка проверки: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED не доказал исправление прав.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"336bfad03f4600ddc0d8996bdbbebeae51cfa0c1e2281ae1a838b8b4d8bc7dfd\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-36-L149\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 149,\\n \\\"title\\\": \\\"Quality check: Storage block needs integrity and pve03 capacity coverage review.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"a11f29a5da892017620722200a35bdde4c0fe5aa591de4331bfbee0851e38266\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-37-L154\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 154,\\n \\\"title\\\": \\\"Coverage gap: pve03_staging missing from disk-space health coverage.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"d2fded36e2e512289b08c31b7fc6872e4efbf61c06e2840e5a2dfef9a8e989f8\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-38-L158\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 158,\\n \\\"title\\\": \\\"Quality check: Service Dependency Map block needs integrity review.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"79cffb917505e112ac9507770da4068e145c75d2278d50d1034bb594b1da5062\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-39-L162\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 162,\\n \\\"title\\\": \\\"Ошибка проверки: post deep audit BAD_MARKER scan caught prose false positive.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"6dc6e561ff4c49c39a8d2f7fdac4fbfd9d2b6295212185791fb9091f644a2824\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-40-L166\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 166,\\n \\\"title\\\": \\\"Ошибка/вывод: VM150 Mail cloud single-file upload failed with 413.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"d7e096eac4cbc2b602601af11ce42bb716d28142a278203aefb65699b5fd4b9c\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-N-41-L170\\\",\\n \\\"kind\\\": \\\"numbered\\\",\\n \\\"source_line\\\": 170,\\n \\\"title\\\": \\\"Ошибка: post-backup-pass-slo-reconciliation checked pve01 router health on edge-vm.\\\",\\n \\\"summary\\\": \\\"\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"44ceb289e0963e0e97c6e8019e93120ad638f5648e8596b987e7d413eaeb59aa\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L31-ПРАВИЛО-ДЛЯ-СПРАВОЧНИКА\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 31,\\n \\\"title\\\": \\\"Правило для справочника\\\",\\n \\\"summary\\\": \\\"Не генерировать большой справочник через интерактивную вставку. Следующий справочник делать только маленькими append-блоками или через файл, созданный вне интерактивного shell. 11. Ошибка: считать offhost OK после failed rsync. Если rsync/scp упал, нельзя проверять latest offhost-файл без сверки имени. Проверка должна подтвердить именно новый архив, например дату 2026_06_30-00_20_59. Старый OFFHOST_ZSTD_OK не закрывает новый backup. 12. Ошибка: широкий secret-поиск по /opt/stacks дал шум. Не искать Cloudflare/SMTP сразу по /opt/stacks, MkDocs site, backups и HTML-отчётам.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"2c69d96d9f906cc88ab3158941ba041ca57ddee1e03554fad588eea96041930a\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L174-ASSISTANT-COMMAND-BATCHING-RULE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 174,\\n \\\"title\\\": \\\"ASSISTANT_COMMAND_BATCHING_RULE_20260630\\\",\\n \\\"summary\\\": \\\"- Rule: combine related infrastructure checks, proof generation, reference updates and validation scans into one safe batch command when practical. - Do not split work into many small commands when the operations are low-risk, read-only, or naturally part of one evidence block. - Still keep mandatory preflight checks, short bounded commands, no large paste, no here-docs, no interactive shell traps, and no sensitive values in output. - Split into smaller commands only when safety, rollback, quoting risk, or troubleshooting clarity requires it.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"73ef09d525c334b9874616d5bafdf17d4898a815d35e736e33df0b83790f6539\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L180-HOME-PORTAL-BASE64-APPLY-FAILURE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 180,\\n \\\"title\\\": \\\"HOME_PORTAL_BASE64_APPLY_FAILURE_20260630\\\",\\n \\\"summary\\\": \\\"- Prior home-portal apply attempt 312 failed because a large base64 Python payload decoded as invalid UTF-8. - Rule: avoid large opaque base64 script payloads for homelab changes; use transparent bounded shell/perl/python commands instead.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"4964d3ff556bb377d7c9d0cb4d487afabc0d8ac69c7fb2a5fccb5ff077a8a478\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L184-HOMELAB-COMMAND-SAFETY-HARDENING-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 184,\\n \\\"title\\\": \\\"HOMELAB_COMMAND_SAFETY_HARDENING_20260630\\\",\\n \\\"summary\\\": \\\"- Rule: do not use large opaque payloads, generated base64 scripts, or hidden multi-line script blobs for homelab changes. - Use transparent bounded shell/perl/python commands, or create a temporary readable script and validate it before execution. - Any script used for changes must pass a syntax/parse check before it is allowed to modify files or restart services. - Any Traceback, UnicodeDecodeError, SyntaxError, Permission denied, command-not-found, nonzero critical RC, or failed write marker must force REVIEW or FAILED, never OK. - Success requires both runtime health and content-specific post-checks: expected files changed, expected cards exist, duplicates absent, service/container alive, and no new critical log pattern. - HTTP 200 alone is not a success condition for configuration changes. - If an apply step fails, the next command must explicitly record the failed attempt and then run a corrected transparent apply command.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"463e648fd9e6d671debe81d12b455563460bc669983e083ef5f0f3b4df2f208b\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L193-HOME-PORTAL-API-ERROR-CLOSURE-RULE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 193,\\n \\\"title\\\": \\\"HOME_PORTAL_API_ERROR_CLOSURE_RULE_20260630\\\",\\n \\\"summary\\\": \\\"- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200. - If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed. - External informational widgets such as weather/Open-Meteo must not block the service launcher portal.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"1a011c6be8607fdacb9571baa017241dcb441a0025f985ecce3d0937a4f7e3b5\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L198-HOME-PORTAL-COMPLEX-QUOTING-FAILURE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 198,\\n \\\"title\\\": \\\"HOME_PORTAL_COMPLEX_QUOTING_FAILURE_20260630\\\",\\n \\\"summary\\\": \\\"- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('. - Cause: command was too complex and fragile due to nested shell/perl/python quoting. - Rule: do not use deeply nested one-liners for config edits; first run short state checks, then apply small transparent edits.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"862820180d4dbc46a845af7c9ec8847aeb21cda3fd7634c0d2afd9b3d14d5d8b\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L203-HOME-PORTAL-ACTIVE-CONFIG-SCOPE-RULE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 203,\\n \\\"title\\\": \\\"HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RULE_20260630\\\",\\n \\\"summary\\\": \\\"- Rule: Homepage cleanup validation must check active config files only, not logs or backup files. - Active files are services.yaml, bookmarks.yaml, widgets.yaml, settings.yaml and docker.yaml under /opt/stacks/homepage/config. - Matches in /logs/ or *.bak* are historical evidence only and must not make cleanup REVIEW unless the active config still contains the target.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"6bfd4c200f43d77ecd18a1c61c377b4c0588a028fb0ca6d565b2361d16a74967\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L208-HOME-PORTAL-LINK-VALIDATION-NPMPLUS-DEFAULT-RULE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 208,\\n \\\"title\\\": \\\"HOME_PORTAL_LINK_VALIDATION_NPMPLUS_DEFAULT_RULE_20260630\\\",\\n \\\"summary\\\": \\\"- Rule: Homepage link validation must not treat HTTP 200 as success if the body is the NPMPlus default “Congratulations” / host-not-set-up page. - Green siteMonitor dots can be false positives when NPMPlus returns its default vhost page. - A portal card is openable only when DNS, reverse-proxy host mapping, and target content are all valid.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"90363c457faef01bbe65d3e246073788fb426212371de346152c7776d246413b\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L213-ROUTER-CLI-STDIN-APPLY-FAILURE-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 213,\\n \\\"title\\\": \\\"ROUTER_CLI_STDIN_APPLY_FAILURE_20260630\\\",\\n \\\"summary\\\": \\\"- Context: attempted Netcraze router ACL apply through SSH stdin/multiline for Homepage Moscow Router monitor fix. - Evidence: 391_MOSCOW_ROUTER_EDGE_VM_WEBADMIN_ACL_APPLY_PROOF.txt. - Failure: STDIN_SHOW_OK=no, APPLY_SKIPPED=stdin_show_failed, CONFIG_HAS_EDGE_5080=no, CONFIG_HAS_EDGE_5083=no. - Rule: do not apply Netcraze config through stdin/multiline SSH; prove CLI input mode first.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"7a151b61221247f7e5ecc6488283190d82b489caa9309675c08baded25d7fc6f\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L219-ROUTER-CLI-PROBE-STDIN-CONSUMPTION-ERROR-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 219,\\n \\\"title\\\": \\\"ROUTER_CLI_PROBE_STDIN_CONSUMPTION_ERROR_20260630\\\",\\n \\\"summary\\\": \\\"- Context: ACL syntax read-only probe loop executed only one command because ssh consumed the loop stdin. - Evidence: 393_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_PROOF.txt showed COMMAND_COUNT=1. - Fix pattern: use ssh -n or redirect SSH stdin away from the command-list loop for all future SSH-in-loop probes.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c801838e0f61bbd4f111bc30b21d530e69124b4448859e0e5f28a93cf0f01fec\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L224-ROUTERBACKUP-READONLY-CLI-PERMISSION-LIMIT-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 224,\\n \\\"title\\\": \\\"ROUTERBACKUP_READONLY_CLI_PERMISSION_LIMIT_20260630\\\",\\n \\\"summary\\\": \\\"- Context: routerbackup user can run show running-config but cannot execute configuration commands such as access-list. - Evidence: 394_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_FIXED_PROOF.txt. - Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli]. - Rule: do not attempt ACL/config changes as routerbackup; use admin/operator router access or router Web UI for write changes.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"1d8075c128dbd4b8f34929055e7c4b7c8cfda8f44f0b7874f04afbcc745d5d82\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L230-PROOF-SUMMARY-EXTRACTION-BLANK-20260630\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 230,\\n \\\"title\\\": \\\"PROOF_SUMMARY_EXTRACTION_BLANK_20260630\\\",\\n \\\"summary\\\": \\\"- Context: proof 395 tried to read COMMAND_COUNT/ZERO_RC_COUNT from proof files 393/394, but those summary fields were printed to console and were not persisted inside the proof files. - Evidence: 395_ROUTER_CLI_ERROR_REGISTER_UPDATE_PROOF.txt has blank P393_COMMAND_COUNT/P394_COMMAND_COUNT/P394_ZERO_RC_COUNT fields. - Rule: when creating follow-up proof summaries, derive counts from persisted proof body lines such as CMD_INDEX and CMD_RC, or write summary fields into the proof file itself.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"1fd6a207e921a706e6d8d3e80984d0436e67192bb56d03c91d20f6b33eba15cd\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L235-ROUTER-ACL-UI-RULE-DELETE-AND-RESTORE-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 235,\\n \\\"title\\\": \\\"ROUTER_ACL_UI_RULE_DELETE_AND_RESTORE_20260701\\\",\\n \\\"summary\\\": \\\"- Context: while moving Moscow Router monitor rules to _WEBADMIN_Bridge1, two _WEBADMIN_Bridge0 Nextcloud Talk TURN rules were accidentally removed. - Deleted/restored rules: Home [PRIVATE_IP]/24 to Nextcloud [PRIVATE_IP] TCP/UDP 3478. - Verification proof: 399_ROUTER_ACL_RESTORE_AFTER_UI_PROOF.txt. - Rule: after manual router Web UI ACL edits, compare _WEBADMIN_Bridge0, _WEBADMIN_Bridge1 and _WEBADMIN_GigabitEthernet1 before continuing.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"973a47bd8e8f388786157b6d50b53de9fc96b31e582a9645f3cf4189b74ac4fe\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L241-ROUTER-MONITOR-NODE-LOOKUP-PROBE-BUG-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 241,\\n \\\"title\\\": \\\"ROUTER_MONITOR_NODE_LOOKUP_PROBE_BUG_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 400 tested Moscow router HTTPS monitor from Homepage container with custom Node lookup callback. - Failure: NODE_HTTPS_RESOLVE_ERROR=ERR_INVALID_IP_ADDRESS and NODE_HTTPS_RESOLVE_INSECURE_ERROR=ERR_INVALID_IP_ADDRESS. - Impact: Node HTTPS resolve subtest is invalid; curl --resolve results remain valid. - Rule: do not rely on custom Node lookup callback until separately tested; prefer curl --resolve or simple direct Node HTTP checks.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"97151454d5ffda69d45b802ed15dc82057b02632e141b3729fa8ad47a959c556\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L247-ROUTER-HTTP-SERVICE-FORBIDDEN-FROM-EDGE-VM-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 247,\\n \\\"title\\\": \\\"ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701\\\",\\n \\\"summary\\\": \\\"- Context: Moscow Router Homepage monitor after ACL fix. - Evidence: proofs 399, 400, 401. - Observed: edge-vm TCP to [PRIVATE_IP]:5080/5083 is OK, but router HTTP/HTTPS returns 403 for all tested paths. - Impact: Homepage siteMonitor cannot be made green via direct router HTTP URL until router web service allows the Bridge1/Proxmox/protected segment or an explicit safe monitor endpoint is used. - Rule: do not keep changing ACL for this symptom; ACL is already correct. Fix service access scope or use a dedicated monitor endpoint.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"b0de3053a46b06841cbaf6ea5945be49ad8d451c7c8453d899d01a9f520cc384\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L254-PY-COMPILE-PYC-PERMISSION-ERROR-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 254,\\n \\\"title\\\": \\\"PY_COMPILE_PYC_PERMISSION_ERROR_20260701\\\",\\n \\\"summary\\\": \\\"- Context: installing edge-vm Moscow router health endpoint. - Mistake: assistant used `python3 -m py_compile` on a script installed under /usr/local/sbin as root. - Actual impact: non-root syntax check attempted to create /usr/local/sbin/__pycache__ and failed with Permission denied. - Recovery: validate syntax with `ast.parse` instead of py_compile, because it does not write .pyc files. - Rule: do not use py_compile against root-owned system paths from an unprivileged user.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"683a661513a8d46a89bf6492c8f12564f6f9fe262ce16b0f6487f63c0f2666f0\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L261-ROUTER-HOME-IP-DOCKER-ROUTE-CONFLICT-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 261,\\n \\\"title\\\": \\\"ROUTER_HOME_IP_DOCKER_ROUTE_CONFLICT_20260701\\\",\\n \\\"summary\\\": \\\"- Context: Moscow Router Homepage siteMonitor attempt using http://[PRIVATE_IP]:5080. - Evidence: proof 405 showed route to [PRIVATE_IP] via Docker bridge and Homepage container ECONNRESET. - Impact: do not use [PRIVATE_IP] directly as Homepage siteMonitor from edge-vm. - Mitigation: use dedicated edge-vm health endpoint that checks router TCP [PRIVATE_IP]:5080 and returns HTTP 200/503 for Homepage.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"5daeb3cf0b7d3bf64ddf6ca87e958f73d0715672617706dee233fd739fb08c2c\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L267-HOMEPAGE-ROUTER-MOSCOW-YAML-TITLE-SHAPE-MISMATCH-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 267,\\n \\\"title\\\": \\\"HOMEPAGE_ROUTER_MOSCOW_YAML_TITLE_SHAPE_MISMATCH_20260701\\\",\\n \\\"summary\\\": \\\"- Context: applying Moscow Router Homepage siteMonitor health endpoint. - Mistake: assistant apply script matched only property-style cards with `title`, but active Homepage YAML can use service-name-key style like `- Service Name:`. - Evidence: proof 407 returned YAML_CHANGE_COUNT=0 and APPLY_SKIPPED=unexpected_match_count while health endpoint proof 406 was OK. - Recovery: use YAML-aware updater that handles both property-style and service-name-key style cards.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"d7ed6a0ecfb19584319e97626e76661541c675e9125e4bf4ff0a407c9718cdd0\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L273-HOMEPAGE-ROUTER-MOSCOW-APPLY-SCRIPT-SYNTAX-ERROR-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 273,\\n \\\"title\\\": \\\"HOMEPAGE_ROUTER_MOSCOW_APPLY_SCRIPT_SYNTAX_ERROR_20260701\\\",\\n \\\"summary\\\": \\\"- Context: applying Moscow Router Homepage health endpoint. - Mistake: assistant generated Python script with invalid f-string escaping. - Evidence: proof 408 showed SyntaxError in /tmp/homepage-router-moscow-apply-fixed.py. - Actual impact: YAML was not changed, so Homepage green dot could not appear. - Recovery: stop applying blind updater; first discover exact active YAML shape, then apply minimal YAML-aware edit. - Rule: validate generated Python syntax locally before scp/remote execution; avoid f-strings in generated maintenance scripts.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"057b7e036316f6f240cac06b9f46e98d8399df689f82b903cc9ef630afe09f94\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L281-FORUM-PROD-VM160-FIRST-BOOT-SSH255-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 281,\\n \\\"title\\\": \\\"FORUM_PROD_VM160_FIRST_BOOT_SSH255_20260701\\\",\\n \\\"summary\\\": \\\"- Context: clean rebuilt VM160 first boot. - Symptom: PING_OK=yes and TCP22_OK=yes, but SSH_RC=255. - Rule: do not proceed with OS baseline until SSH failure is diagnosed; likely old known_hosts key or cloud-init/root-key issue.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"5e0359dc6b12e39f533e9060876f5c5f830b43cf2dbbe0441d6876fee5f410a3\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L286-FORUM-PROD-VM160-NESTED-HOSTNAME-PROOF-QUOTING-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 286,\\n \\\"title\\\": \\\"FORUM_PROD_VM160_NESTED_HOSTNAME_PROOF_QUOTING_20260701\\\",\\n \\\"summary\\\": \\\"- Context: VM160 first SSH proof after rebuild. - Issue: command substitution $(hostname) inside nested ssh was expanded on pve02 before entering VM160. - Impact: VM_HOSTNAME=pve02 in proof 426 is not valid VM identity evidence. - Rule: for nested SSH identity checks, run literal hostname commands without local command substitution.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c7174e4703945abd82004fea3fa9da0ff3efa1f2a89e6d2315cacf9c27b6d761\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L292-FORUM-PROD-VM160-SWAP-PROOF-QUOTING-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 292,\\n \\\"title\\\": \\\"FORUM_PROD_VM160_SWAP_PROOF_QUOTING_20260701\\\",\\n \\\"summary\\\": \\\"- Context: VM160 swapfile proof 429. - Issue: nested command substitution expanded on pve02, producing invalid SWAPFILE/FSTAB proof lines. - Impact: proof 429 is not valid closure evidence even though swap was active. - Rule: avoid nested $(...) in VM SSH proofs; use literal remote commands and clean proof.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"3dc2ea2b0a54adb8886979de4e01cb5f23704446f5b2198af9bdacc951359e4c\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L298-FORUM-UPLOAD-ARCHIVES-VM-VERIFY-QUOTING-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 298,\\n \\\"title\\\": \\\"FORUM_UPLOAD_ARCHIVES_VM_VERIFY_QUOTING_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 446 copy/check archives inside VM160. - Issue: nested SSH quoting expanded shell variables incorrectly, producing gzip checks against empty .gz and blank TAR_TOP lines. - Impact: SCP_TO_VM_RC=0 and file size list are valid, but GZIP/TAR proof lines in 446 are invalid. - Rule: verify VM archive integrity locally inside forum-prod, not through nested quoted loops.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"0e4a3a2f7c179f46c235e1ffa176b57485c88b4f7e7314bb89b6acb6683fe448\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L304-FORUM-PROD-BULK-IMPORT-PHP85-EMPTY-FRONTEND-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 304,\\n \\\"title\\\": \\\"FORUM_PROD_BULK_IMPORT_PHP85_EMPTY_FRONTEND_20260701\\\",\\n \\\"summary\\\": \\\"- Context: rebuilt VM160, five XenForo forums imported in bulk on PHP 8.5.7. - Symptom: all five forums returned HTTP 200 with empty body; XenForo logged Template public:PAGE_CONTAINER/forum_list is unknown. - Additional failed approach: proof 480 ran xf-dev:rebuild-caches on CodeVipe and returned RC=1, not a valid fix. - Decision: stop bulk repair, treat current VM160 as lab, find a working CodeVipe recipe first, then destroy/rebuild cleanly for all five forums. - Rule: no more mass rebuild/repair commands across all forums until single-forum CodeVipe recipe is proven.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"ea3c74648eece7c2a94559d8f7385ea4a126b399e3a161c5113e69a7b8d05f93\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L311-XENFORO-REQUIREMENTS-DIRECT-DOWNLOAD-403-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 311,\\n \\\"title\\\": \\\"XENFORO_REQUIREMENTS_DIRECT_DOWNLOAD_403_20260701\\\",\\n \\\"summary\\\": \\\"- Context: Tried to download XenForo requirements ZIP directly from xenforo.com inside forum-prod. - Issue: curl returned HTTP 403; unzip then failed because the ZIP was not downloaded. - Evidence: proof 491 showed DOWNLOAD_RC=22, ZIP_LIST_RC=9, UNZIP_RC=9. - Impact: proof 491 is not a valid server compatibility test. - Rule: use operator-uploaded xenforo23-requirements-test.zip from /root/forum-upload instead of relying on direct curl download.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"13164e6750f679553f0bb7cb13f4b895f345dc3f76c978349dfd30b93294206b\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L318-PVE01-UNZIP-MISSING-FOR-XF-REQUIREMENTS-VERIFY-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 318,\\n \\\"title\\\": \\\"PVE01_UNZIP_MISSING_FOR_XF_REQUIREMENTS_VERIFY_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 492 tried to inspect uploaded xenforo23-requirements-test.zip on pve01. - Issue: pve01 does not have unzip installed, so ZIP_TEST_RC=127 was not a ZIP integrity result. - Impact: proof 492 confirmed file presence only, not archive validity. - Rule: verify ZIP with Python zipfile or inside forum-prod rather than installing unzip on Proxmox just for inspection.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"341e814b6c0aab80d0bde631353118c8c04593d918a2ed7b60551d5bc3043148\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L324-FRESH5-DEPLOY-SUCCESS-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 324,\\n \\\"title\\\": \\\"FRESH5_DEPLOY_SUCCESS_20260701\\\",\\n \\\"summary\\\": \\\"- Context: VM160 was destroyed/recreated, then five XenForo forums were deployed from fresh ZIP backups. - Result: proof 513 confirms all five forums locally healthy. - Note: proof 511 initially failed dsmods path detection; proof 512 fixed dsmods by locating public_html via src/XF.php marker. - Rule: future restores should preserve internal_data/code_cache or rebuild compiled template cache before smoke testing.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"f46e9779cb65eebc42015f7042a4f55f0a8aa1e59e647eb29d728e48ed6069b3\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L330-NPMPLUS-SQLITE-PASTE-FAILURE-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 330,\\n \\\"title\\\": \\\"NPMPLUS_SQLITE_PASTE_FAILURE_20260701\\\",\\n \\\"summary\\\": \\\"- Context: NPMplus SQLite inspection command was too complex and a Python fragment was pasted into bash. - Issue: shell entered multiline prompt and produced syntax errors. - Impact: do not trust that SQLite inspection attempt. - Rule: avoid nested SSH heredoc/Python for this task; use simpler commands, generated nginx configs, or uploaded/local script files.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"d1f2dda1b36bc12803264780b71545d7d9b00c4bdb88ef2fa85405e9d27cdf6f\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L336-NPMPLUS-PUBLISH-SCRIPT-WRONG-ENV-SOURCE-525-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 336,\\n \\\"title\\\": \\\"NPMPLUS_PUBLISH_SCRIPT_WRONG_ENV_SOURCE_525_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 525 tried to create forum proxy hosts in NPMplus. - Issue: script sourced /opt/npmplus/.env, but INITIAL_ADMIN_EMAIL/PASSWORD were not present there; they are in the npmplus container env. - Impact: no forum proxy hosts were created by proof 525. - Rule: read NPMplus API login values from docker inspect env internally, never print them.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"58aa4eeee2841a42e270226a408177af9a099631e2a47857c825707b2ac523ff\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L342-NPMPLUS-API-LOGIN-MISSED-HTTPS81-AND-JSON-ESCAPE-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 342,\\n \\\"title\\\": \\\"NPMPLUS_API_LOGIN_MISSED_HTTPS81_AND_JSON_ESCAPE_20260701\\\",\\n \\\"summary\\\": \\\"- Context: NPMplus API login attempts in proofs 526/527 failed. - Issue: commands missed the likely HTTPS admin endpoint on port 81 and built JSON by string interpolation instead of jq escaping. - Impact: no proxy hosts were created by 526/527. - Rule: for NPMplus API use jq-generated JSON and test https://127.0.0.1:81/api/tokens with -k before publish.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"141ce6d2777003569c531b5308e56cffaea92dc113fdb0c9e9ebe45815805101\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L348-NPMPLUS-API-AUTH-UNAVAILABLE-MANUAL-ROUTES-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 348,\\n \\\"title\\\": \\\"NPMPLUS_API_AUTH_UNAVAILABLE_MANUAL_ROUTES_20260701\\\",\\n \\\"summary\\\": \\\"- Context: NPMplus API login attempts failed even though the forum VM and edge-to-forum connectivity are healthy. - Issue: API credentials from container initial env are not accepted by current NPMplus API. - Impact: do not use NPMplus API for this publish path. - Rule: publish forum routes through backed-up manual nginx proxy_host configs, then issue proper certificates after DNS points to edge.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"d5a4f4c6312277e9e2c1487b38c4bf554f4b9618fad9e2c5328fe8324e26a555\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L354-EDGE-CERTBOT-TOKEN-FILE-PERMISSION-535-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 354,\\n \\\"title\\\": \\\"EDGE_CERTBOT_TOKEN_FILE_PERMISSION_535_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 535 attempted DNS-01 certificate issue for five forum domains. - Issue: /tmp/forum_cf_token.env was root-owned mode 600, so debian user could not source it directly. - Impact: certbot ran without token and all certificate attempts failed; DNS cutover must not proceed from proof 535. - Rule: read Cloudflare token via sudo without printing it, then create temporary certbot credentials file and remove it after use.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"4566b4ef97181c31efcb32eefd3ed52633883fc929e98d4607a6c48671434582\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L360-EDGE-CERTBOT-FAILED-DNS-CUTOVER-ABORTED-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 360,\\n \\\"title\\\": \\\"EDGE_CERTBOT_FAILED_DNS_CUTOVER_ABORTED_20260701\\\",\\n \\\"summary\\\": \\\"- Context: edge manual routes are working locally, but certbot DNS-01 certificate issue failed in proof 538. - Issue: all five certbot attempts returned RC=1; proof 539 correctly aborted DNS cutover because edge cert precheck was not ready. - Impact: Cloudflare DNS records were not changed by proof 539; public DNS still points to old IP 87.236.18.45. - Rule: inspect certbot logs and Cloudflare token zone permissions before retrying certificate issuance or DNS cutover.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"7fc8883c0f715d038cc77e38a9088d6cb18290d2456232e9e0a13b1ba094d256\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L366-CLOUDFLARE-TOKEN-NO-ZONE-ACCESS-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 366,\\n \\\"title\\\": \\\"CLOUDFLARE_TOKEN_NO_ZONE_ACCESS_20260701\\\",\\n \\\"summary\\\": \\\"- Context: Cloudflare token discovered on edge verified as a token but could not access forum zones. - Evidence: proof 542 showed all five zones missing and DNS record create probes failed. - Impact: certbot DNS-01 failed and DNS cutover was correctly aborted; public DNS still points to old hosting. - Rule: install a new Cloudflare token with Zone Read and DNS Edit for the five forum zones before retrying certs/DNS.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"3ba4fdbb1dcc5d0beeb215153e4471c19e29f4080ff3d061e77154f44de9a898\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L372-FORUM-PUBLICATION-FINAL-SUCCESS-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 372,\\n \\\"title\\\": \\\"FORUM_PUBLICATION_FINAL_SUCCESS_20260701\\\",\\n \\\"summary\\\": \\\"- Context: five XenForo forums were rebuilt on VM160, routed through edge NPMplus, issued Let’s Encrypt certs using a corrected Cloudflare token, and cut over in DNS. - Result: final public proof 546 passed. - Important corrections: initial NPMplus API route failed; manual Nginx routes were used. Initial Cloudflare token had no zone access; corrected token passed zone/DNS probe.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"bec7980ce2230b8ecf69cfc694cc56bbd3dedc45da55ed2df567e8f41b228638\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L377-FINAL-SNAPSHOT-NAME-TOO-LONG-556-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 377,\\n \\\"title\\\": \\\"FINAL_SNAPSHOT_NAME_TOO_LONG_556_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 556 final health gate passed for all five public forums. - Issue: Proxmox snapshot failed because generated snapshot name exceeded 40 characters. - Evidence: qm snapshot returned snapname value may only be 40 characters long. - Impact: forum health was OK, but proof 556 snapshot step was not completed. - Fix: rerun snapshot with short name.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"c3d65c37278c1e067af0f6dc1d7eb36fa6e75b03113162e66adf06c483bedc76\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L384-CLOUDFLARE-DNS-AUDIT-TXT-SUBSTRING-GAP-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 384,\\n \\\"title\\\": \\\"CLOUDFLARE_DNS_AUDIT_TXT_SUBSTRING_GAP_20260701\\\",\\n \\\"summary\\\": \\\"- Context: prior Cloudflare DNS audit checked web A/AAAA/CNAME state for root/www records. - Issue: it did not deeply inspect TXT/SPF contents for old provider IP substrings. - Evidence: user screenshot showed SPF TXT containing old 87.236.18.* value. - Impact: web routing is OK, but mail-related DNS may still contain stale provider data. - Rule: perform full DNS record audit including TXT/SPF/DKIM/DMARC/MX/CDN and substring search for old IPs before DNS cleanup.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"260c1f016ae9bf9efb5888bb54809e742681b512223781efac0a4cc8d8cdf2c7\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L391-CLOUDFLARE-DNS-CLEANUP-DRYRUN-QUOTE-563-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 391,\\n \\\"title\\\": \\\"CLOUDFLARE_DNS_CLEANUP_DRYRUN_QUOTE_563_20260701\\\",\\n \\\"summary\\\": \\\"- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt. - Impact: proof 563 is invalid and no DNS cleanup was performed by it. - Rule: use copied scripts for Cloudflare DNS mutation instead of fragile inline nested quotes.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"3837def1e15ccdc91af766587f44bd7e98885fb25b88ca3d62e65d13698f62d1\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L396-SPF-DUPLICATE-AFTER-565-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 396,\\n \\\"title\\\": \\\"SPF_DUPLICATE_AFTER_565_20260701\\\",\\n \\\"summary\\\": \\\"- Context: SPF cleanup command 565 attempted to replace stale SPF records. - Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained. - Impact: domains temporarily had duplicate SPF records, which is invalid for mail validation. - Fix: delete all SPF TXT records for the five forum zones, then create exactly one v=spf1 -all TXT per zone.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"4a4b1cba0ac945aef40797821d7a9a774728f30fb5e56c34599575e1b5b40ba8\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L402-MSMTP-SECRET-SOURCE-PARSE-ERROR-576-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 402,\\n \\\"title\\\": \\\"MSMTP_SECRET_SOURCE_PARSE_ERROR_576_20260701\\\",\\n \\\"summary\\\": \\\"- Context: proof 576 installed msmtp but sendmail auth test failed. - Issue: [SENSITIVE_PATH] was sourced as shell, but SMTP password contained shell-special characters; source failed and auth used an invalid/truncated secret path. - Impact: msmtp package installed, but mail sending was not proven working. - Rule: never source SMTP secret files containing arbitrary passwords; store password base64 and use msmtp passwordeval helper accessible to www-data.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"6288f1fcabcdeda1884be858bea052ff9dbb35a65054f5cff6315a350dd7ef12\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L408-FORUM-MSMTP-MAIL-TRANSPORT-STILL-FAILING-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 408,\\n \\\"title\\\": \\\"FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701\\\",\\n \\\"summary\\\": \\\"- Context: attempted to fix msmtp config with passwordeval helper. - Issue: one or more sendmail/PHP mail tests still failed. - Impact: XenForo mail sending is not yet proven. - Next step: verify SMTP credential/password and provider policy for noreply@pvepro.ru.\\\",\\n \\\"rule_like\\\": false,\\n \\\"signature\\\": \\\"31d614a3bb69ec05a33a5be14388347a438e13f1e28a7ec58207273159e00eae\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L414-SMTP-PASSWORD-EXPOSED-ROTATE-REQUIRED-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 414,\\n \\\"title\\\": \\\"SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701\\\",\\n \\\"summary\\\": \\\"- Context: SMTP password was exposed in terminal output during failed msmtp setup. - Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line. - Impact: treat that SMTP password as compromised. - Required action: rotate the mailbox/app password in the mail provider panel before any further SMTP testing. - Rule: never store arbitrary passwords as shell-sourceable KEY=value files; use non-printing secret storage only.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"88ae0aeb13d948e6f2cb2d83dc2a9464e4de1805cdb4271cf78f3d315f7095c9\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L421-NETBIRD-MAILCOW-REPORTED-DOWN-AFTER-SMTP-INCIDENT-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 421,\\n \\\"title\\\": \\\"NETBIRD_MAILCOW_REPORTED_DOWN_AFTER_SMTP_INCIDENT_20260701\\\",\\n \\\"summary\\\": \\\"- Context: user reported NetBird and Mailcow became unavailable after forum mail-transport work. - Impact: treat as active incident until service reachability and container/VM state are proven. - Rule: do not change DNS, firewall, proxy, mail, NetBird, or Mailcow configs before read-only triage proof.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"b0eef26c0503e63d8a8c5a294ea1e124ab5e6ac8936d2ccaf23cd9b74fc5ebf3\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L426-FORUM-SMTP-FAILED-CONFIG-DISABLED-AFTER-MAILCOW-BAN-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 426,\\n \\\"title\\\": \\\"FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701\\\",\\n \\\"summary\\\": \\\"- Context: forum-prod SMTP/msmtp test used invalid or compromised credentials and triggered Mailcow netfilter warnings/ban for 95.84.154.183. - Impact: forum mail transport is not proven and must remain disabled until SMTP password is rotated and tested once carefully. - Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth. - Rule: do not test SMTP auth again until a fresh rotated mailbox/app password is available.\\\",\\n \\\"rule_like\\\": true,\\n \\\"signature\\\": \\\"70dc70d8c15e3f30611c45123bf2c3b27c86bf66d703df3e4232057d380801d6\\\"\\n },\\n {\\n \\\"id\\\": \\\"ERR-H-L432-FORUM-OLD-CODEVIPE-CLOUD-BACKUP-STALE-AFTER-FRESH5-20260701\\\",\\n \\\"kind\\\": \\\"heading\\\",\\n \\\"level\\\": 2,\\n \\\"source_line\\\": 432,\\n \\\"title\\\": \\\"FORUM_OLD_CODEVIPE_CLOUD_BACKUP_STALE_AFTER_FRESH5_20260701\\\",\\n \\\"summary\\\": \\\"- Context: pve02 codevipe-cloud-backup.timer existed before fresh five-forum rebuild. - Issue: old script points to /root/backups/codevipe and old remote path app/codevipe/forum-prod/local. - Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory" +} diff --git a/runtime/latest.txt b/runtime/latest.txt index 1202733..e8402e6 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,3891 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=CONTEXT-AUTO-[PRIVATE_IP]701Z +COMMAND_ID=CONTEXT-SOURCES-FULL-REFRESH-363 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-context +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=eb33834ec5df95efdbe55c4264ce02cbc69fcf90de62d457948a23d1e1efda04 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=9482ef04e0765ed01d6de068ebba4ef30fb0e133871b6cd45c4af4906d299122 +SANITIZED_OUTPUT_SHA256=3d9b0a662d873becba40d548b6854862d5ad518285238a73362573ff85988751 +OUTPUT_BEGIN +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/test +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/mktemp +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/rm +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/curl +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/jq +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/stat +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/sha256sum +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/awk +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/date +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/cat +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/printf +DETAIL=LOCAL_DEPENDENCY=PASS:/usr/bin/grep +DETAIL=STAGE=SOURCE_FETCH +DETAIL=IMMUTABLE_SOURCE=errors-index|SIZE=146628|SHA256=a13ca59d5fa1ad39b813bad3115d7b8cf8c5c68935fc813dbd792757ed421e24|STATUS=PASS +DETAIL=IMMUTABLE_SOURCE=workflow-contract|SIZE=1926|SHA256=1378a2c0d6dec6b09e3ea7dfced65fed1e1c38065baa6bba8bfd4c89e23e2c78|STATUS=PASS +DETAIL=IMMUTABLE_SOURCE=double-check-rule|SIZE=268|SHA256=7e5221154959bb0f435b3cfb955e6671d3c361ecc7a99625f04252cc74ebaa4b|STATUS=PASS +DETAIL=FULL_CONTEXT_BUNDLE_BEGIN +FULL_CONTEXT_BUNDLE_SCHEMA=1 +GENERATED_AT_UTC=2026-07-23T14:23:43Z +REFERENCE_FILE=/etc/pve/31_HOMELAB_REFERENCE.md +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +REFERENCE_SIZE=164602 +ERROR_REGISTER_FILE=/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +ERROR_REGISTER_SIZE=79246 +AUTO_CONTEXT_ID=CONTEXT-AUTO-20260723T141652Z +AUTO_CONTEXT_COMMIT=fcf2ed2a6b11016ec6441a014f7832a86c51f793 +AUTO_CONTEXT_URL=https://git.gram1.ru/homelab-admin/homelab-public-context/raw/commit/fcf2ed2a6b11016ec6441a014f7832a86c51f793/runtime/history/CONTEXT-AUTO-20260723T141652Z.txt +AUTO_CONTEXT_SHA256=acd6a206ec5eb45ed50c2aae1cee02d4683dd8675e5c2d88190cefcd2446dada +AUTO_CONTEXT_SIZE=29049 +LATEST_JSON_AUTHORITATIVE=NO +DETAIL=SECTION_REFERENCE_REGISTER_BEGIN +HOMELAB REFERENCE +GENERATED=2026-06-29T21:45:29+03:00 +AUDIT_DIR=/root/cluster-audit-20260629T201245 + +CORE_CLUSTER_CONFIG + +keyboard: en-us +migration: secure,network=[PRIVATE_IP]/24 + +FINAL_CLASSIFICATION +MISSING_EXPECTED_PREFIX_COUNT=0 +STRICT_POSSIBLE_SECRET_COUNT=0 +STRICT_SECRET_SCAN_OK +HEALTH_WARN_ERROR_COUNT=14 +NOTE exclude_from_reference_or_delete_later=05_edge_compose_safe.tgz +NOTE forum-prod VMID160 is not in homelab-nightly-all; it has separate local/cloud backup chain +NOTE edge-vm disk scsi1 backup=0 risk must be documented +NOTE Corosync remains on [PRIVATE_IP]/12/13; migration network is [PRIVATE_IP]/24 secure +NOTE edge health WARN/ERROR items should be documented as known current states + +HEALTH_NOT_OK +HEALTH_NOT_OK item=adguard-rewrite-sync.txt line=STATUS=ERROR TS=20260629T152553Z TYPE=adguard-rewrite-sync REASON=URLError SECRET_PRINTED=REDACTED +HEALTH_NOT_OK item=backup-restore-coverage-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=backup-restore-dashboard.txt line=STATUS=WARN TS=2026-06-29T18:10:09Z ITEMS=80 NOT_OK=13 +HEALTH_NOT_OK item=container-image-lifecycle-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=cron-job-monitoring-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=direct-heartbeat-pilot-readiness.txt line=STATUS=WARN +HEALTH_NOT_OK item=healthchecks-heartbeat-coverage.txt line=STATUS=WARN +HEALTH_NOT_OK item=healthchecks-job-coverage.txt line=STATUS=WARN +HEALTH_NOT_OK item=healthchecks-job-monitoring-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=homepage-service-catalog-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=ingress-dns-route-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=job-monitoring-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=observability-health-surface-slo.txt line=STATUS=WARN +HEALTH_NOT_OK item=security-vulnerability-slo.txt line=STATUS=WARN + +EVIDENCE_FILES +00_cluster_overview.txt 3544 bytes +00_HOMELAB_ASSISTANT_ERROR_REGISTER.md 2566 bytes +01_pve01_audit.txt 28554 bytes +02_pve02_audit.txt 24786 bytes +03_pve03_audit.txt 16842 bytes +04_edge_vm_basic.txt 34506 bytes +05_edge_compose_redacted.txt 24228 bytes +05_edge_compose_safe.tgz.QUARANTINED.txt 354 bytes +05_edge_compose_tar_errors.txt 166 bytes +06_ACCESS_AND_ERROR_RULES.md 13742 bytes +06_edge_npmplus_routes_safe.txt 17350 bytes +07_edge_systemd_backup_audit.txt 20694 bytes +08_edge_scripts_redacted.txt 198622 bytes +09_forum_prod_basic.txt 14413 bytes +10_forum_codevipe_xenforo_audit.txt 3861 bytes +11_forum_backup_audit_redacted.txt 5952 bytes +12_pve02_codevipe_cloud_backup_audit.txt 6277 bytes +13_pve01_systemd_backup_audit.txt 60109 bytes +14_pve01_scripts_redacted.txt 246474 bytes +15_pve01_ct_110_audit.txt 5049 bytes +15_pve01_ct_112_audit.txt 4843 bytes +16_pve02_ct_111_audit.txt 4758 bytes +16_pve02_ct_113_audit.txt 4843 bytes +17_nextcloud_vm_audit.txt 13293 bytes +18_nextcloud_backup_restore_pve01_audit.txt 4269 bytes +19_pve02_host_automation_audit.txt 10642 bytes +19_pve03_host_automation_audit.txt 7632 bytes +20_AUDIT_INDEX_AND_SECRET_SCAN.txt 11935 bytes +21_proxmox_cluster_config_audit.txt 10855 bytes +22_internal_2_5g_network_inventory.txt 8036 bytes +23_cluster_internal_network_configured.txt 747 bytes +24_cluster_internal_network_speedtest.txt 54665 bytes +25_cluster_internal_network_migration_enabled.txt 1816 bytes +26_migration_network_pvesh_verify.txt 1210 bytes +27_migration_network_canonical_verify.txt 956 bytes +28_dns_configs_redacted.txt 15559 bytes +29_health_summary_all_nodes.txt 39051 bytes +30_FINAL_AUDIT_GAPS_WARNINGS_SECRET_CLASSIFICATION.txt 4366 bytes +31_HOMELAB_REFERENCE.md 1805 bytes +datacenter.cfg.before-canonical-migration-20260629T211046 63 bytes +datacenter.cfg.before-migration-network-20260629T210710 16 bytes + + + +РУССКОЕ_РЕЗЮМЕ_КЛАСТЕРА +- Кластер: homelab, 3 узла, quorum OK. +- LAN: pve01=[PRIVATE_IP], pve02=[PRIVATE_IP], pve03=[PRIVATE_IP]. +- Внутренняя сеть: pve01-int=[PRIVATE_IP], pve02-int=[PRIVATE_IP], pve03-int=[PRIVATE_IP]. +- Миграции Proxmox: migration: secure,network=[PRIVATE_IP]/24. +- Corosync остаётся на [PRIVATE_IP]/12/13; не переносить без отдельного rollback-плана. + +РИСКИ_И_ДЕЙСТВИЯ +- VM160 forum-prod не входит в ночной Proxmox backup. +- У VM130 edge-vm есть риск: дополнительный диск backup=0. +- Нужно ротировать ранее засвеченный Cloudflare token. +- Желательно ротировать XenForo SMTP пароль, который был выведен до редактирования. +- Проверить adguard-rewrite-sync: STATUS=ERROR, REASON=URLError. +- Проверить backup-restore-dashboard: STATUS=WARN, NOT_OK=13. + +ДОСТУПЫ_КРАТКО +- Proxmox: ssh root@pve01, ssh root@pve02, ssh root@pve03. +- Edge VM: ssh debian@[PRIVATE_IP], использовать sudo, root-login не использовать. +- Nextcloud VM: ssh debian@[PRIVATE_IP]. +- Forum-prod: сначала ssh root@pve02, затем ssh -i [SENSITIVE_PATH] root@[PRIVATE_IP]. + +НАГРУЗКИ_И_СЕРВИСЫ +- CT110 dns1 pve01 [PRIVATE_IP] AdGuard Home. +- CT111 dns2 pve02 [PRIVATE_IP] AdGuard Home. +- CT112 unbound1 pve01 [PRIVATE_IP] Unbound. +- CT113 unbound2 pve02 [PRIVATE_IP] Unbound. +- VM130 edge-vm pve03 [PRIVATE_IP] Docker ingress/app host. +- VM150 Nextcloud pve01 [PRIVATE_IP] Nextcloud AIO. +- VM160 forum-prod pve02 [PRIVATE_IP] CodeVipe/XenForo. + +BACKUP_КРАТКО +- Ночной Proxmox backup включает VMID 110,111,112,113,130,150. +- VM160 forum-prod исключён из ночного Proxmox backup и держится на app-level local/cloud backup. +- Edge VM имеет vzdump/offhost/restore evidence; риск backup=0 по дополнительному диску остаётся. +- Nextcloud VM150 имеет Proxmox backup и restore-proof evidence. +- CodeVipe forum-prod имеет локальный backup каждые 6 часов и cloud backup с pve02 каждые 6 часов. + +БЕЗОПАСНОСТЬ_КРАТКО +- Финальная строгая проверка audit-файлов: STRICT_POSSIBLE_SECRET_COUNT=0. +- Redacted DNS configs сохранены в 28_dns_configs_redacted.txt. +- Справочник 31 проверен: REFERENCE_ASSIGNMENT_SECRET_HITS=0. + +## FINAL_CLOSURE_20260630_CRITICAL_TAILS + +- VM160 backup: closed, nightly includes VM160, manual backup ZSTD_OK. +- VM130 backup: closed, scsi1 backup=1, manual backup ZSTD_OK, exact offhost ZSTD_OK, old local backup removed. +- AdGuard rewrite sync: closed, STATUS=OK, timer active/enabled. +- Cloudflare token: live NPMplus token rotated, API verify OK, old exposed token externally confirmed revoked. +- XenForo SMTP: password rotated, JSON valid, STARTTLS SMTP AUTH OK. +- Final audit: unredacted secret strict scan OK. +- Proof index: 85_FINAL_CLOSURE_PROOF.txt and 87_FINAL_CLOSURE_PROOF_WITH_EXTERNAL_CONFIRMATION.txt. + +## FINAL_DASHBOARD_RUNTIME_OK_20260630 + +- Runtime dashboard: STATUS=OK, ITEMS=69, NOT_OK=0. +- systemd failed units: 0 loaded units listed. +- SLO/coverage WARN files: moved out of runtime dashboard into slo-coverage-backlog-index.txt. +- Backlog index: STATUS=OK, BACKLOG_WARN_COUNT=12, RUNTIME_BLOCKER=NO. +- Final proof: 103_FINAL_ALL_GREEN_DASHBOARD_PROOF.txt. +## ROUTER_NETCRAZE_ULTRA_NC1812_20260630 + +Источник: self-test Netcraze Ultra NC-1812 от 2026-06-30. Секреты Wi-Fi, пароли и токены в справочник не вносятся. + +### Паспорт +- Model: Netcraze Ultra. +- Device description: Netcraze Ultra (NC-1812). +- Hostname: Netcraze-9202. +- Workgroup/domain: WORKGROUP. +- Timezone: Europe/Moscow. +- NTP: master. +- NDNS/caption: ndns-domain. +- Локальный домен/сертификат в логах: aleisaevn.netcraze.pro. +- Firmware/NDM из self-test: 5.01.C.0.0-1, built 2026-06-17. +- sharing-config version: 2.06.1. +- Components auto-update: disabled. +- Auto-update channel: draft. +- Auto-update schedule0: start 05:00, stop 06:00. +- EasyConfig: disabled. +- zram: enabled. +- IPv4 forwarding: enabled. +- IPv6 forwarding: enabled. +- conntrack max entries: 32768. +- TCP established timeout: 1200. +- TCP fin timeout: 30. +- TCP keepalive: 120. + +### WAN и резервный интернет +- Main WAN: GigabitEthernet1, renamed ISP, description Ростел. +- WAN security-level: public. +- WAN addressing: DHCP. +- WAN MTU: 1500. +- WAN global priority: 700. +- WAN ping-check profile: default. +- Ping-check hosts from log/profile: ya.ru, vk.com, dzen.ru over TLS/443. +- WAN DHCP observed in router log: 95.84.154.183/23, gateway 95.84.154.1. +- ISP DNS observed in router log: 77.37.251.33, 77.37.255.30. +- Backup/mobile WAN: CdcEthernet0, description SIM. +- CdcEthernet0 USB device-id: 12d1 14dc. +- CdcEthernet0 security-level: public. +- CdcEthernet0 addressing: DHCP. +- CdcEthernet0 global priority: 350. +- CdcEthernet0 observed IP: [PRIVATE_IP]/24 via [PRIVATE_IP]. + +### LAN / VLAN / bridge +- Home bridge: Bridge0 renamed Home. +- Home description: Основная. +- Home security-level: private. +- Home IP: [PRIVATE_IP]/24. +- Home includes: GigabitEthernet0/Vlan1, AccessPoint, AccessPoint_5G, XGigabitEthernet0. +- Proxmox bridge: Bridge1. +- Proxmox bridge IP: [PRIVATE_IP]/24. +- Proxmox security-level: protected. +- Proxmox includes: GigabitEthernet0/Vlan50, XGigabitEthernet0/Vlan50. +- Port 1: GigabitEthernet0/0, access VLAN 50. +- Port 2: GigabitEthernet0/1, access VLAN 1 + trunk VLAN 50. +- Port 3: GigabitEthernet0/2, access VLAN 1 + trunk VLAN 50. +- Port 4: GigabitEthernet0/3, access VLAN 1 + trunk VLAN 50. +- Port 5: XGigabitEthernet0/0, XGigabitEthernet0/Vlan50 enabled. + +### Wi-Fi +- SSID: N9202. +- 2.4 GHz: WifiMaster0, AccessPoint. +- 2.4 GHz compatibility: BGN+AX+BE. +- 2.4 GHz channel width: 40-below. +- 5 GHz: WifiMaster1, AccessPoint_5G. +- 5 GHz compatibility: AN+AC+AX+BE. +- 5 GHz channel width: 160. +- Auto channel rescan: 00:00 interval 1 hour. +- Encryption: WPA2 + WPA3. +- WMM: enabled. +- Beamforming: enabled. +- TWT: enabled. +- DL/UL MU-MIMO: enabled. +- DL/UL OFDMA: enabled. +- Spatial reuse: enabled. +- Band steering: disabled. +- Extra AP interfaces: present but down. +- MWS WLAN Home: bands 0 and 1, bind Home, SSID N9202, encryption wpa2+3. + +### DHCP +- Main DHCP pool: _WEBADMIN. +- DHCP range: [PRIVATE_IP]-[PRIVATE_IP]. +- Default router: [PRIVATE_IP]. +- DHCP DNS: [PRIVATE_IP], [PRIVATE_IP]. +- Lease: 25200 seconds. +- Bound interface: Home. +- Guest AP pool: _WEBADMIN_GUEST_AP enabled. + +### Static DHCP reservations +- [PRIVATE_IP] -> 00:e0:70:62:98:fb comp01. +- [PRIVATE_IP] -> 60:3d:61:d4:57:ea Yandex-Ambi-Lamp. +- [PRIVATE_IP] -> cc:4b:73:58:9b:c0 Yandex.Station. +- [PRIVATE_IP] -> 38:8a:06:5b:eb:40 Мой смарт. +- [PRIVATE_IP] -> a0:9d:c1:84:7b:98 Приставка комната. +- [PRIVATE_IP] -> ac:ba:c0:51:d3:ca Алиса наша комната. +- [PRIVATE_IP] -> 54:f1:5f:f3:cd:14 Приставка кухня. +- [PRIVATE_IP] -> ac:ba:c0:50:71:94 Алиса кухня. +- [PRIVATE_IP] -> 24:4b:fe:d8:8d:2c Коля смарт ROG-Phone3. +- [PRIVATE_IP] -> 00:2b:70:9a:a3:cf Коля смарт S21-FE. +- [PRIVATE_IP] -> 00:08:22:34:26:fc Коля SPIDER-10 первый. +- [PRIVATE_IP] -> 6c:c7:ec:93:be:dc Света смарт. +- [PRIVATE_IP] -> 00:1e:8f:b0:d5:af Принтер Canon ч/б. +- [PRIVATE_IP] -> d8:43:ae:68:a3:a1 Коля комп. +- [PRIVATE_IP] -> 80:1f:12:69:24:e6 Сигнализация. +- [PRIVATE_IP] -> 00:2b:70:9a:9e:99 Мой смарт S21-FE. +- [PRIVATE_IP] -> 74:eb:80:2e:aa:4e Света планшет Galaxy Tab S4. +- [PRIVATE_IP] -> 84:47:09:6a:cc:ca PVE01. +- [PRIVATE_IP] -> 6c:1f:f7:28:72:4f PVE02. +- [PRIVATE_IP] -> 08:26:ae:3b:73:61 PVE03. +- [PRIVATE_IP] -> bc:24:11:30:d4:0f AdGuard 1. +- [PRIVATE_IP] -> bc:24:11:5b:7e:d8 AdGuard 2. +- [PRIVATE_IP] -> bc:24:11:f4:c5:d8 WireGuard. +- [PRIVATE_IP] -> bc:24:11:e1:f3:3c NPMplus / edge-vm. +- [PRIVATE_IP] -> bc:24:11:e1:9a:25 Nextcloud. + +### NAT / port forwarding +- ISP tcp/80 -> bc:24:11:e1:f3:3c, NPMplus. +- ISP tcp/443 -> bc:24:11:e1:f3:3c, NPMplus. +- ISP tcpudp/3478 -> bc:24:11:e1:9a:25, Nextcloud Talk TURN. +- Home tcp/51820 -> bc:24:11:f4:c5:d8, WireGuard vpn1. + +### ACL / firewall +- isolate-private enabled. +- _WEBADMIN_Bridge0 permits Home-to-Proxmox access for DNS1/DNS2, AdGuard UI, NPMplus HTTP/HTTPS/UI, Nextcloud AIO/Talk TURN, Proxmox SSH/8006 and ICMP from admin PC. +- _WEBADMIN_GigabitEthernet1 permits admin PC [PRIVATE_IP] outbound TCP 80/443. +- Hotspot policy: Home permit, Bridge1 permit; known hosts explicitly permitted. + +### Router management +- HTTP port: 5080. +- HTTPS port: 5083. +- HTTP/HTTPS security-level: private. +- SSH port: 2222. +- SSH security-level: private. +- Telnet port configured: 2323, security-level private; router log shows Telnet disabled later on 2026-06-26. +- Lockout policy for HTTP/Telnet/SSH: 5 15 3. +- Cloud control2 security-level: private. +- Admin tags: cli, http, cifs, printers, opt. +- SFTP denied for admin in log; SSH CLI works. + +### Router services +- service dhcp enabled. +- service dns-proxy enabled. +- service igmp-proxy enabled. +- service http enabled. +- service cifs enabled. +- service ssh enabled. +- service ntp enabled. +- DNS proxy rebind protection: auto. +- mDNS reflector: disabled. +- UPnP LAN: Home. +- DLNA interface: Home. +- CIFS share: NTFS-edgtobxf, volume 6F98787822C01361:, automount, permissive. +- USB storage seen in log: Huawei modem/storage, sdb1 swap around 2G, sdb2 NTFS volume NTFS-edgtobxf. + +### Router automation warning +- Netcraze SSH CLI is not a normal POSIX shell. +- Logs contain failed commands: while, unset, follow. +- Automation must use router CLI syntax, not bash syntax. +## EXTERNAL_SERVICES_UPS_NETBIRD_MAIL_SCRIPTS_20260630 + +### UPS / NUT +- UPS hardware discovered on pve01 USB: Cyber Power System CP1500 AVR UPS, USB ID 0764:0501. +- pve01 role: NUT server + monitor. +- pve01 enabled NUT units: nut-driver-enumerator, nut-monitor, nut-server, nut-driver.target, nut.target. +- pve02 role: NUT monitor/client. +- pve03 role: NUT monitor/client. +- edge-vm: no UPS/NUT/APCUPSD integration discovered. +- UPS detailed proof: 107_UPS_DISCOVERY_AND_SETTINGS.txt and 109_UPS_NUT_DETAILED_STATUS_AND_CONFIGS.txt. +- Secrets from /etc/nut configs must remain redacted in audit output. + +### NetBird +- NetBird service is active on pve01, pve02, pve03 and edge-vm. +- NetBird version observed: daemon 0.73.2, CLI 0.73.2. +- pve01: FQDN pve01.netbird.selfhosted, IPv4 100.100.131.41/16. +- pve02: FQDN pve02.netbird.selfhosted, IPv4 100.100.7.2/16. +- pve03: FQDN pve03.netbird.selfhosted, IPv4 100.100.34.141/16. +- edge-vm: FQDN edge-vm.netbird.selfhosted, IPv4 100.100.60.182/16. +- Interface: wt0, type Kernel. +- WireGuard port: 51820. +- Management/Signal: Connected. +- Relays: 2/2 available. +- SSH Server through NetBird: Disabled. +- Observed peers count on listed hosts: 6/9 Connected. +- Inventory proof: 108_EXTERNAL_SERVICES_NETBIRD_MAIL_SCRIPTS_INVENTORY.txt. + +### Forum mail / SMTP +- forum-prod host: forum-prod.gram1.ru, VM IP [PRIVATE_IP]. +- Local MTA services were not discovered by the first inventory command; forum mail is treated as XenForo external SMTP, not local Postfix/Dovecot. +- XenForo SMTP rotation proof: 80_xenforo_smtp_rotation_closed.txt. +- SMTP auth proof: 79_forum_xenforo_smtp_auth_openssl_no_crlf_no_secret.txt. +- Safe current SMTP inventory: 110_XENFORO_MAIL_EXTERNAL_SMTP_SAFE_INVENTORY.txt. +- SMTP password must never be printed; store only host, port, SSL mode, auth flag, username length, password length and AUTH OK proof. + +### Scripts / automations +- Full script/unit inventory proof: 108_EXTERNAL_SERVICES_NETBIRD_MAIL_SCRIPTS_INVENTORY.txt. +- edge-vm owns most application health, dashboard, ingress, backup, NetBox, NPMplus, certificate, Trivy and vulnerability jobs. +- pve01 owns many backup/offhost/restore/health/security/NetBird VPS/rclone/sops/scrutiny jobs. +- pve02 owns CodeVipe cloud backup, Vaultwarden isolated restore drill, smartctl textfile and staging/rclone helpers. +- pve03 owns smartctl textfile, cluster internal IP, NetBird and staging/rclone helpers. +- All scripts must be treated as managed operational surface; every future edit needs proof file in the active cluster-audit directory. +## UPS_NUT_DETAILED_CONFIG_20260630 + +- USB discovery on pve01: Cyber Power System, Inc. CP1500 AVR UPS, USB ID 0764:0501. +- NUT logical UPS name: cyberpower. +- NUT-reported device model: BR1000ELCD. +- NUT manufacturer: CPS. +- Driver: usbhid-ups. +- NUT driver version: 2.8.1. +- CyberPower HID data version: 0.8. +- NUT USB vendorid/productid: 0764/0501. +- NUT server node: pve01. +- NUT server mode: MODE=netserver. +- NUT server listeners: 127.0.0.1:3493 and [PRIVATE_IP]:3493. +- NUT clients: pve02 and pve03 in MODE=netclient. +- pve01 monitor role: MONITOR cyberpower@localhost master. +- pve02 monitor role: MONITOR cyberpower@[PRIVATE_IP] slave. +- pve03 monitor role: MONITOR cyberpower@[PRIVATE_IP] slave. +- Current UPS status at inventory time: OL. +- Battery charge: 100%. +- Battery warning threshold: 20%. +- Battery low threshold: 10%. +- Runtime estimate: 2544 seconds. +- Runtime low threshold: 300 seconds. +- Battery type: PbAcid. +- Battery voltage: 12.9V nominal 12V. +- Input voltage: 221.0V nominal 230V. +- Output voltage: 221.0V. +- UPS load: 11%. +- Nominal real power: 600W. +- Beeper: disabled. +- Shutdown delay: 20 seconds. +- Start delay: 30 seconds. +- Shutdown command on monitored nodes: /sbin/shutdown -h +0. +- NUT timing: POLLFREQ=5, POLLFREQALERT=5, HOSTSYNC=15, DEADTIME=30, FINALDELAY=5. +- Powerdown flag: /etc/killpower. +- Secrets in /etc/nut/upsd.users and upsmon.conf are intentionally redacted in proofs. +- Proof file: 109_UPS_NUT_DETAILED_STATUS_AND_CONFIGS.txt. + +## XENFORO_EXTERNAL_SMTP_CURRENT_20260630 + +- Host: forum-prod.gram1.ru, VM IP [PRIVATE_IP]. +- Local MTA units: none discovered in inventory output. +- Mail mode: XenForo external SMTP, not local Postfix/Dovecot. +- SMTP_HOST=mail.pvepro.ru +- SMTP_PORT=587 +- SMTP_SSL=false +- SMTP_AUTH=login +- USERNAME_LEN=17 +- PASSWORD_LEN=30 +- SECRET_PRINTED=NO +- SMTP AUTH proof: 79_forum_xenforo_smtp_auth_openssl_no_crlf_no_secret.txt. +- Rotation closure proof: 80_xenforo_smtp_rotation_closed.txt. +## TIMERS_AND_SCRIPT_HASHES_REFERENCE_20260630 + +- Full timers and script SHA256 inventory: 114_TIMERS_AND_SCRIPT_HASHES_INVENTORY.txt. +- edge-vm timers cover runtime dashboard, Paperless guard, external canary, health metrics, restore drill index, AdGuard rewrite sync, NPMplus cert expiry, NetBox backup/sync, retention, cluster daily status, vulnerability and Trivy scans, ingress hardening and NPMplus admin bind. +- pve01 timers cover VPN/NetBird health, health metrics, smartctl, disk space, MkDocs refresh, VPS identity audit, storage capacity, quality gate, evidence catalog, backup freshness, docker health, Filebrowser backup/offhost/restore, NPMplus/Kuma backup, NetBird VPS backup/offhost, Authentik/Gitea/Vaultwarden backup, SOPS secret coverage, mail cloud upload/restore, Immich/Memos/Paperless backup/offhost/restore, auto backup, edge-vm vzdump, secret sanity. +- pve02 timers cover smartctl, CodeVipe cloud backup, dpkg-db backup and Vaultwarden isolated restore drill. +- pve03 timers cover smartctl and dpkg-db backup. +- Script hashes were captured for /usr/local/sbin and /usr/local/bin on edge-vm, pve01, pve02 and pve03. +- Future script edits must create a new proof file and refresh 114_TIMERS_AND_SCRIPT_HASHES_INVENTORY.txt. +## PROXMOX_CLUSTER_NODE_STORAGE_REFERENCE_20260630 + +### Cluster +- Cluster name: homelab. +- Nodes: 3. +- Quorum: OK / Quorate Yes. +- Expected votes: 3. +- Quorum threshold: 2. +- Transport: knet. +- Secure auth: on. +- Corosync membership: [PRIVATE_IP] pve01, [PRIVATE_IP] pve02, [PRIVATE_IP] pve03. +- Proxmox VE: pve-manager 9.2.3 on all three nodes. +- Debian: 13 / trixie on all three nodes. +- Kernel: 7.0.12-1-pve on all three nodes. +- Datacenter migration config: secure, network=[PRIVATE_IP]/24. +- Internal migration/cluster helper IPs: pve01 [PRIVATE_IP], pve02 [PRIVATE_IP], pve03 [PRIVATE_IP]. + +### Storage policy +- Storage local: dir /var/lib/vz, content iso/vztmpl/backup/import. +- Storage local-lvm: lvmthin pool data, content rootdir/images. +- pve01 local usage at inventory: 32.56%, local-lvm 17.50%. +- pve02 local usage at inventory: 12.86%, local-lvm 11.65%. +- pve03 local usage at inventory: 35.79%, local-lvm 64.84%. +- pve01 staging LV: /mnt/staging, 300G. +- pve02 staging LV: /mnt/staging, 150G. +- pve03 staging LV: /mnt/staging, 200G. +- pve03 local-lvm is the most constrained active VM storage because VM130 has 96G OS disk plus 150G media disk. + +### Nightly Proxmox backup job +- Job: homelab-nightly-all. +- Schedule: 03:30. +- Mode: snapshot. +- Compression: zstd. +- Storage: local. +- Enabled: yes. +- Mail notification: failure. +- Included VMIDs: 110,111,112,113,130,150,160. +- Retention: keep-daily=7, keep-weekly=4, keep-monthly=3. + +### Node pve01 +- FQDN: pve01.gram1.ru. +- LAN IP: [PRIVATE_IP]/24. +- Internal migration IP: [PRIVATE_IP]/24. +- NetBird IP: 100.100.131.41/16. +- Bridge: vmbr0 over nic0. +- CPU: Intel Core i9-12950HX, 24 logical CPUs. +- RAM: 31Gi. +- Disk: Lexar SSD NQ7A1 1TB. +- Main active workloads: CT110 dns1, CT112 unbound1, VM150 nextcloud. + +### Node pve02 +- FQDN: pve02.gram1.ru. +- LAN IP: [PRIVATE_IP]/24. +- Internal migration IP: [PRIVATE_IP]/24. +- NetBird IP: 100.100.7.2/16. +- Bridge: vmbr0 over nic2. +- CPU: Intel N100, 4 logical CPUs. +- RAM: 15Gi. +- Disk: SK800-1TB. +- Main active workloads: CT111 dns2, CT113 unbound2, VM160 forum-prod. + +### Node pve03 +- FQDN: pve03.gram1.ru. +- LAN IP: [PRIVATE_IP]/24. +- Internal migration IP: [PRIVATE_IP]/24. +- NetBird IP: 100.100.34.141/16. +- Bridge: vmbr0 over nic1. +- CPU: Intel Core i7-4900MQ, 8 logical CPUs. +- RAM: 31Gi. +- Disk: Samsung SSD 870 EVO 500GB. +- Main active workload: VM130 edge-vm. + +### Proof +- Full raw inventory: 120_PROXMOX_CLUSTER_NODE_STORAGE_NETWORK_INVENTORY.txt. +## PROXMOX_VM_CT_INVENTORY_REFERENCE_20260630 + +### CT110 dns1 +- Type: LXC. +- Node: pve01. +- Hostname/name: dns1. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:F9:3C:E1. +- Role: AdGuard Home DNS primary. +- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameserver: [PRIVATE_IP]. +- Onboot: yes. +- Startup order: 30. +- Unprivileged: yes. +- Backup job: included in homelab-nightly-all. + +### CT111 dns2 +- Type: LXC. +- Node: pve02. +- Hostname/name: dns2. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:CF:3F:66. +- Role: AdGuard Home DNS secondary. +- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameserver: [PRIVATE_IP]. +- Onboot: yes. +- Startup order: 30. +- Unprivileged: yes. +- Backup job: included in homelab-nightly-all. + +### CT112 unbound1 +- Type: LXC. +- Node: pve01. +- Hostname/name: unbound1. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:22:A6:0D. +- Role: Unbound recursive resolver primary. +- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameserver: [PRIVATE_IP]. +- Onboot: yes. +- Startup order: 20. +- Unprivileged: yes. +- Backup job: included in homelab-nightly-all. + +### CT113 unbound2 +- Type: LXC. +- Node: pve02. +- Hostname/name: unbound2. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:9E:AF:BE. +- Role: Unbound recursive resolver secondary. +- Resources: 1 core, 512MiB RAM, 256MiB swap, 8G rootfs. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameserver: [PRIVATE_IP]. +- Onboot: yes. +- Startup order: 20. +- Unprivileged: yes. +- Backup job: included in homelab-nightly-all. + +### VM130 edge-vm +- Type: QEMU VM. +- Node: pve03. +- Name: edge-vm. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:E1:F3:3C. +- User: debian. +- Role: edge application host / reverse proxy / monitoring / backup automation host. +- Resources: 4 cores, 12GiB RAM. +- Disks: scsi0 96G OS, scsi1 150G media/data. +- scsi1 backup flag: backup=1. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameservers: [PRIVATE_IP], [PRIVATE_IP]. +- QEMU guest agent: enabled. +- Onboot: yes. +- Startup order: 40. +- Backup job: included in homelab-nightly-all. +- Important note: VM130 has exact offhost backup proof after scsi1 backup=1. + +### VM150 nextcloud +- Type: QEMU VM. +- Node: pve01. +- Name: nextcloud. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:E1:9A:25. +- User: debian. +- Role: Nextcloud AIO. +- Resources: 4 cores, 8GiB RAM. +- Disk: scsi0 64G. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameservers: [PRIVATE_IP], [PRIVATE_IP]. +- QEMU guest agent: enabled. +- Onboot: yes. +- Startup order: 50. +- Backup job: included in homelab-nightly-all. + +### VM160 forum-prod +- Type: QEMU VM. +- Node: pve02. +- Name: forum-prod. +- IP: [PRIVATE_IP]/24. +- MAC: BC:24:11:B6:45:ED. +- User: ops. +- Role: CodeVipe / XenForo production forum. +- Resources: 2 cores, 4GiB RAM. +- Disk: scsi0 80G. +- Network: vmbr0, gateway [PRIVATE_IP]. +- Nameserver in VM config: 1.1.1.1. +- QEMU guest agent: enabled. +- Onboot: yes. +- Startup order: 30. +- Backup job: included in homelab-nightly-all. +- Important note: VM160 manual backup proof exists and VM160 is included in nightly job. + +### Proof +- Redacted config inventory: 121_PROXMOX_VM_CT_CONFIG_INVENTORY_REDACTED.txt. +## DNS_INGRESS_CERTIFICATES_REFERENCE_20260630 + +### DNS chain +- Home DHCP gives clients DNS servers: [PRIVATE_IP] and [PRIVATE_IP]. +- dns1: CT110, AdGuard Home, IP [PRIVATE_IP]. +- dns2: CT111, AdGuard Home, IP [PRIVATE_IP]. +- unbound1: CT112, IP [PRIVATE_IP], listens on [PRIVATE_IP]:5335 and 127.0.0.1:5335. +- unbound2: CT113, IP [PRIVATE_IP], listens on [PRIVATE_IP]:5335 and 127.0.0.1:5335. +- dns1 upstream: [PRIVATE_IP]:5335. +- dns2 upstream: [PRIVATE_IP]:5335. +- AdGuard bootstrap DNS: 1.1.1.1 and 9.9.9.9. +- AdGuard filtering_enabled=true, rewrites_enabled=true, protection_enabled=true. +- AdGuard ratelimit=20. +- Unbound access model: localhost allowed, matching AdGuard IP allowed, rest of [PRIVATE_IP]/24 refused. +- Unbound do-ip6: no. +- Unbound private-address includes [PRIVATE_IP]/16, [PRIVATE_IP]/12 and [PRIVATE_IP]/8. + +### AdGuard rewrites +- turn.gram1.ru -> [PRIVATE_IP]. +- git.gram1.ru -> [PRIVATE_IP]. +- dozzle.gram1.ru -> [PRIVATE_IP]. +- paper.gram1.ru -> [PRIVATE_IP]. +- memos.gram1.ru -> [PRIVATE_IP]. +- photos.gram1.ru -> [PRIVATE_IP]. +- auth.gram1.ru -> [PRIVATE_IP]. +- backup.gram1.ru -> [PRIVATE_IP]. +- Rewrite sync health proof: adguard-rewrite-sync STATUS=OK, ORIGIN_COUNT=8, REPLICA_COUNT=8, CHANGED=0. +- Proof files: 127_ADGUARD_UNBOUND_DNS_SAFE_INVENTORY.txt and 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt. + +### Router ingress +- Public WAN router forwards TCP/80 and TCP/443 to NPMplus on edge-vm, [PRIVATE_IP]. +- Router forwards TCP/UDP 3478 to Nextcloud Talk TURN on [PRIVATE_IP]. +- Router forwards Home TCP/51820 to WireGuard host [PRIVATE_IP]. +- NPMplus admin listener is bound to localhost on edge-vm, 127.0.0.1:81; public disabled legacy host npm.gram1.ru exists but enabled=0. + +### NPMplus runtime +- Host: edge-vm, [PRIVATE_IP]. +- Container: npmplus, image zoeyvid/npmplus:2026-06-17-b1, healthy at inventory time. +- Socket proxy: lscr.io/linuxserver/socket-proxy:latest, 127.0.0.1:2375. +- Database: /opt/npmplus/npmplus/database.sqlite. +- DB integrity: ok. +- Public listen ports on edge-vm: 0.0.0.0:80 and 0.0.0.0:443 by nginx/NPMplus. +- NPMplus admin: 127.0.0.1:81. +- Secret rule: Cloudflare DNS API token exists only inside NPMplus certificate metadata and must never be printed. + +### Public NPMplus proxy hosts +- nc.gram1.ru -> http://[PRIVATE_IP]:11000, cert=17, ssl_forced=1, enabled=1. +- uptime.gram1.ru -> http://127.0.0.1:3001, cert=35, ssl_forced=1, enabled=1. +- gotify.gram1.ru -> http://127.0.0.1:8082, cert=35, ssl_forced=1, enabled=1. +- vault.gram1.ru -> http://127.0.0.1:8083, cert=21, ssl_forced=1, enabled=1. +- dockge.gram1.ru -> http://127.0.0.1:5001, cert=26, ssl_forced=1, enabled=1. +- home.gram1.ru -> http://127.0.0.1:3000, cert=27, ssl_forced=1, enabled=1. +- git.gram1.ru -> http://127.0.0.1:3002, cert=29, ssl_forced=1, enabled=1. +- dozzle.gram1.ru -> http://127.0.0.1:9999, cert=30, ssl_forced=1, enabled=1. +- paper.gram1.ru -> http://127.0.0.1:8010, cert=30, ssl_forced=1, enabled=1. +- memos.gram1.ru -> http://127.0.0.1:5230, cert=30, ssl_forced=1, enabled=1. +- photos.gram1.ru -> http://127.0.0.1:2283, cert=30, ssl_forced=1, enabled=1. +- auth.gram1.ru -> http://127.0.0.1:9000, cert=30, ssl_forced=1, enabled=1. +- backup.gram1.ru -> http://[PRIVATE_IP]:9101, cert=31, ssl_forced=1, enabled=1. + +### VPN NPMplus proxy hosts +- kuma.vpn.gram1.ru -> http://127.0.0.1:3001, cert=32. +- homepage.vpn.gram1.ru -> http://127.0.0.1:3000, cert=32. +- gitea.vpn.gram1.ru -> http://127.0.0.1:3002, cert=32. +- filebrowser.vpn.gram1.ru -> http://127.0.0.1:8085, cert=32. +- auth.vpn.gram1.ru -> http://127.0.0.1:9000, cert=32. +- actual-budget.vpn.gram1.ru -> http://127.0.0.1:5006, cert=32. +- grafana.vpn.gram1.ru -> http://127.0.0.1:3003, cert=32. +- homebox.vpn.gram1.ru -> http://127.0.0.1:7745, cert=32. +- mealie.vpn.gram1.ru -> http://127.0.0.1:9001, cert=32. +- n8n.vpn.gram1.ru -> http://127.0.0.1:5678, cert=32. +- netbox.vpn.gram1.ru -> http://127.0.0.1:8000, cert=32. +- node-red.vpn.gram1.ru -> http://127.0.0.1:1880, cert=32. +- beszel.vpn.gram1.ru -> http://127.0.0.1:8090, cert=32. +- it-tools.vpn.gram1.ru -> http://127.0.0.1:8084, cert=32. +- karakeep.vpn.gram1.ru -> http://127.0.0.1:3030, cert=32. +- linkding.vpn.gram1.ru -> http://127.0.0.1:9091, cert=32. +- minio.vpn.gram1.ru -> http://127.0.0.1:9003, cert=32. +- ntfy.vpn.gram1.ru -> http://127.0.0.1:8055, cert=32. +- searxng.vpn.gram1.ru -> http://127.0.0.1:8888, cert=32. +- syncthing.vpn.gram1.ru -> http://127.0.0.1:8384, cert=32. +- alertmanager.vpn.gram1.ru -> http://127.0.0.1:9093, cert=32. +- prometheus.vpn.gram1.ru -> http://127.0.0.1:9090, cert=32. +- homeassistant.vpn.gram1.ru -> http://127.0.0.1:8123, cert=32. +- npmplus.vpn.gram1.ru -> https://[PRIVATE_IP]:81, cert=32. +- proxmox.vpn.gram1.ru -> https://[PRIVATE_IP]:8006, cert=32. +- dns1.vpn.gram1.ru -> http://[PRIVATE_IP]:8080, cert=32. +- dns2.vpn.gram1.ru -> http://[PRIVATE_IP]:8080, cert=32. +- checks.vpn.gram1.ru -> http://127.0.0.1:8015, cert=32. +- vikunja.vpn.gram1.ru -> http://127.0.0.1:8016, cert=32. +- bookstack.vpn.gram1.ru -> http://127.0.0.1:8017, cert=32. +- pdf.vpn.gram1.ru -> http://127.0.0.1:8018, cert=32. +- jellyfin.vpn.gram1.ru -> http://127.0.0.1:8019, cert=32. +- audiobooks.vpn.gram1.ru -> http://127.0.0.1:8020, cert=32. +- books.vpn.gram1.ru -> http://127.0.0.1:8021, cert=32. + +### NPMplus certificates +- cert=17: nc.gram1.ru, expires 2026-09-12 21:58:35. +- cert=21: vault.gram1.ru, expires 2026-09-13 13:43:23. +- cert=26: dockge.gram1.ru, expires 2026-09-13 14:51:59. +- cert=27: home.gram1.ru, expires 2026-09-13 16:43:44. +- cert=29: git.gram1.ru, expires 2026-09-13 17:36:55. +- cert=30: dozzle.gram1.ru, paper.gram1.ru, memos.gram1.ru, photos.gram1.ru, auth.gram1.ru, expires 2026-09-13 23:12:00. +- cert=31: backup.gram1.ru, expires 2026-09-14 05:04:53. +- cert=32: *.vpn.gram1.ru and vpn.gram1.ru, expires 2026-09-15 23:18:29. +- cert=35: uptime.gram1.ru and gotify.gram1.ru, expires 2026-09-19 21:21:59. +- Deleted/old cert rows observed: cert=1 nc.gram1.ru, cert=18 vault.gram1.ru. + +### Proof +- NPMplus schema/listen inventory: 126_DNS_INGRESS_CERT_NPMPLUS_SAFE_INVENTORY.txt. +- NPMplus exact proxy/cert rows: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt. +- AdGuard/Unbound inventory: 127_ADGUARD_UNBOUND_DNS_SAFE_INVENTORY.txt. +- AdGuard upstream/rewrites and Unbound rules: 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt. +## EDGE_DOCKER_STACKS_REFERENCE_20260630 + +### Runtime +- Host: edge-vm, IP [PRIVATE_IP]. +- Docker Server version: 29.6.0. +- Docker Compose version: v5.1.4. +- Primary stack root: /opt/stacks. +- Additional compose roots: /opt/npmplus-compose, /opt/gotify-compose, /opt/uptime-kuma-compose, /opt/vaultwarden-compose, /opt/dockge-compose. +- Public ingress terminates through NPMplus on ports 80/443. +- Most app containers expose only 127.0.0.1 ports and are published through NPMplus. +- NPMplus uses host networking. +- socket-proxy exposes Docker API read-oriented surface on 127.0.0.1:2375 and socket-proxy_net. +- Secret values are not stored in the reference. Only .env/secret file paths are inventoried. + +### Compose projects observed +- actual-budget. +- audiobookshelf. +- authentik. +- beszel. +- blackbox-exporter. +- bookstack. +- cadvisor. +- calibre-web. +- crowdsec. +- diun. +- dockge-compose. +- dozzle. +- filebrowser. +- gitea. +- gotify-compose. +- healthchecks. +- homeassistant. +- homebox. +- homepage. +- immich. +- it-tools. +- jellyfin. +- karakeep. +- kopia. +- linkding. +- loki-alloy. +- mealie. +- memos. +- minio. +- n8n. +- netbox. +- node-red. +- npmplus. +- ntfy. +- observability-lite. +- paperless. +- searxng. +- socket-proxy. +- stirling-pdf. +- syncthing. +- uptime-kuma-compose. +- vaultwarden-compose. +- vikunja. + +### Critical app groups +- Ingress/security: npmplus, socket-proxy, crowdsec. +- Identity/secrets: authentik, vaultwarden. +- Monitoring/alerting: uptime-kuma, gotify, healthchecks, observability-lite, loki-alloy, blackbox-exporter, cadvisor, node-exporter, dozzle, diun. +- Data apps: immich, paperless, memos, gitea, netbox, actual-budget, homebox, mealie, n8n, node-red, vikunja, bookstack. +- Files/media: nextcloud external VM, immich, filebrowser, syncthing, jellyfin, audiobookshelf, calibre-web, stirling-pdf, minio. +- Admin/dashboard: homepage, dockge, netbox, dozzle, NPMplus VPN admin. + +### Notable local ports +- Homepage: 127.0.0.1:3000. +- Uptime Kuma: 127.0.0.1:3001. +- Gitea: 127.0.0.1:3002. +- Grafana: 127.0.0.1:3003. +- Actual Budget: 127.0.0.1:5006. +- n8n: 127.0.0.1:5678. +- Paperless: 127.0.0.1:8010. +- Healthchecks: 127.0.0.1:8015. +- Vikunja: 127.0.0.1:8016. +- BookStack: 127.0.0.1:8017. +- Stirling PDF: 127.0.0.1:8018. +- Jellyfin: 127.0.0.1:8019. +- Audiobookshelf: 127.0.0.1:8020. +- Calibre-Web: 127.0.0.1:8021. +- ntfy: 127.0.0.1:8055. +- Gotify: 127.0.0.1:8082. +- Vaultwarden: 127.0.0.1:8083. +- IT-Tools: 127.0.0.1:8084. +- Filebrowser: 127.0.0.1:8085. +- Beszel: 127.0.0.1:8090. +- Prometheus: 127.0.0.1:9090. +- Linkding: 127.0.0.1:9091. +- Alertmanager: 127.0.0.1:9093. +- Node exporter: 127.0.0.1:9100. +- Blackbox exporter: 127.0.0.1:9115. +- Syncthing UI: 127.0.0.1:8384. +- Loki: 127.0.0.1:3100. +- Alloy UI: 127.0.0.1:12345. +- Home Assistant: host network, 0.0.0.0:8123. +- NPMplus admin: 127.0.0.1:81. +- NPMplus public ingress: 0.0.0.0:80 and 0.0.0.0:443. + +### Secret/env inventory policy +- Full values from .env, secrets.yaml, secretkey.txt and token files must never be copied into reference. +- Reference may list paths only. +- Full path inventory is in 133_EDGE_DOCKER_STACKS_SAFE_INVENTORY.txt. +- Normalized runtime map is in 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt. + +### Proof +- Raw Docker/stacks inventory: 133_EDGE_DOCKER_STACKS_SAFE_INVENTORY.txt. +- Normalized container map: 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt. +## BACKUP_RESTORE_OFFHOST_CATALOG_REFERENCE_20260630 + +### Global backup model +- Primary Proxmox backup job: homelab-nightly-all. +- Schedule: daily 03:30. +- Mode: snapshot. +- Compression: zstd. +- Storage: local. +- Retention: keep-daily=7, keep-weekly=4, keep-monthly=3. +- Included VMIDs: 110,111,112,113,130,150,160. +- Backup health dashboard: STATUS=OK, ITEMS=69, NOT_OK=0 at 2026-06-30T10:10:06Z. +- Restore drill index: STATUS=OK, RESTORE_PROOFS=16/16. +- SLO coverage is tracked separately from runtime dashboard. It had WARN backlog in backup-restore-coverage-slo with APP_TOTAL=43, GREEN=3, YELLOW=21, RED=19, MISSING_BACKUP_SIGNAL=31, MISSING_RESTORE_SIGNAL=28, MISSING_TIMER=31. +- Runtime OK means operational backup/restore checks are green; it does not mean every P1/P2 application has full backup+restore automation coverage. + +### Proxmox vzdump catalog +- CT110 dns1: included in homelab-nightly-all, local backup on pve01. +- CT111 dns2: included in homelab-nightly-all, local backup on pve02. +- CT112 unbound1: included in homelab-nightly-all, local backup on pve01. +- CT113 unbound2: included in homelab-nightly-all, local backup on pve02. +- VM130 edge-vm: included in homelab-nightly-all, local backup on pve03. +- VM150 nextcloud: included in homelab-nightly-all, local backup on pve01. +- VM160 forum-prod: included in homelab-nightly-all, local backup on pve02. +- VM130 also has dedicated edge-vm-vzdump backup/offhost/restore health proofs. +- VM160 has manual backup proof and nightly job inclusion proof. +- VM130 scsi1 backup flag is enabled after correction: scsi1 backup=1. + +### Edge VM / VM130 full-image protection +- edge-vm-vzdump-backup: STATUS=OK, archive size about 28.2G, SHA256 recorded. +- edge-vm-vzdump-offhost: STATUS=OK, destination pve02 /mnt/staging/offhost/edge-vm-vzdump-from-pve03. +- edge-vm-vzdump-restore: STATUS=OK, zstd and vma verification OK. +- mail-cloud-edge-vm: STATUS=OK, recurring chunked upload, 53 parts, download verification enabled. +- Retention for edge-vm cloud upload: RETENTION_KEEP=4. +- Edge restore proof offhost: STATUS=OK, ITEMS=14, OK=14. + +### Mail/cloud critical backups +- mail-cloud-critical: STATUS=OK, recurring P0 critical baseline, remote OK 2/2. +- Latest observed critical bundle: p0-critical-baseline-20260630T034644Z.tar.gz. +- Critical bundle size at inventory: 1087208837 bytes. +- Critical retention: RETENTION_KEEP=14. +- mail-cloud-restore-drill: STATUS=OK, CRITICAL_OK=2, EDGE_OK=1. +- vaultwarden isolated restore drill on pve02: STATUS=OK, uses cloud critical baseline plus pve02 offhost component and vaultwarden archive. + +### NPMplus / Kuma / ingress config backups +- npmplus-kuma-config-backup: STATUS=OK. +- Archive: /mnt/staging/npmplus-kuma-config-backups/snapshots/npmplus-kuma-config-*.tar.gz. +- NPMplus DB integrity: OK. +- Kuma DB integrity: OK. +- NPM proxy count in health proof: 47. +- Required VPN routes in health proof: 18. +- Wildcard VPN cert check: NPM_CERT32_WILDCARD=1. +- npmplus-kuma-config-offhost: STATUS=OK to pve02. +- npmplus-kuma-config-restore: STATUS=OK with DB integrity checks. +- npmplus restore proof also exists from app backup quality checks. + +### DNS / AdGuard / Unbound protection +- AdGuard rewrite sync: STATUS=OK, ORIGIN_COUNT=8, REPLICA_COUNT=8, CHANGED=0. +- dns1/dns2/unbound1/unbound2 are protected by Proxmox vzdump. +- AdGuard rewrite sync plan stored under /var/lib/homelab-private/adguard-rewrite-sync. +- Router self-test config is stored as uploaded/generated reference evidence, not yet automated as recurring router backup. + +### Auth / secrets / identity apps +- Vaultwarden backup: STATUS=OK, archive under /mnt/staging/vaultwarden-backups/snapshots. +- Vaultwarden offhost: STATUS=OK to pve02. +- Vaultwarden restore: STATUS=OK, DB integrity OK, tables counted. +- Vaultwarden isolated restore drill: STATUS=OK on pve02. +- Authentik backup: STATUS=OK, archive under /mnt/staging/authentik-backups/snapshots. +- Authentik offhost: STATUS=OK to pve02. +- Authentik restore: STATUS=OK, tables/users/applications/proxy providers checked. +- Secret values must remain out of all reference files; only archive paths, sizes and SHA256 are recorded. + +### Git / documentation / inventory apps +- Gitea backup: STATUS=OK, archive under /mnt/staging/gitea-backups/snapshots. +- Gitea offhost: STATUS=OK to pve02. +- Gitea restore: STATUS=OK, DB integrity OK, tables counted. +- NetBox backup: STATUS=OK, PostgreSQL dump under /var/lib/homelab-private/pre-update-backups/netbox-*. +- NetBox offhost: STATUS=OK to pve02. +- NetBox restore dry-run: STATUS=OK, restore container checked. +- Latest NetBox health proof observed: devices=7, sites=1, prefixes=3, IP addresses=12, virtual machines=49. + +### Document / notes / personal data apps +- Paperless backup: STATUS=OK, latest snapshot under /mnt/staging/paperless-backups/snapshots. +- Paperless offhost: STATUS=OK to pve02. +- Paperless restore dry-run: STATUS=OK, pg_restore list and restore tables checked. +- Memos backup: STATUS=OK, latest snapshot under /mnt/staging/memos-backups/snapshots. +- Memos offhost: STATUS=OK to pve02. +- Memos restore proof exists from app restore quality checks. +- Actual Budget backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof. +- Linkding backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof. +- Karakeep backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof. +- Mealie backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof. +- n8n backup: STATUS=OK, offhost SHA256/listing OK, restore not attempted in observed latest proof. + +### Files / media / sync apps +- Immich media backup: STATUS=OK, latest snapshot under /mnt/staging/immich-media-backups/snapshots. +- Immich media offhost: STATUS=OK to pve02. +- Immich media restore: STATUS=OK, local/offhost manifest match. +- Immich full consistency backup: STATUS=OK, includes PostgreSQL dump and media archive, offhost path present, restore dry-run OK. +- Nextcloud restore proof: STATUS=OK, OCC OK, DB_DUMP OK, DATA_MANIFEST OK, restore to production NO. +- Nextcloud restore proof is copied offhost. +- Filebrowser backup: STATUS=OK. +- Filebrowser offhost: STATUS=OK. +- Filebrowser restore validation: STATUS=OK. +- Jellyfin restore: STATUS=OK from app backup quality checks. +- Audiobookshelf restore: STATUS=OK from app backup quality checks. +- Calibre-Web restore: STATUS=OK from app backup quality checks. +- Syncthing is inventoried in Docker/stacks; dedicated backup signal not confirmed in this catalog. + +### Home/admin/utility apps +- Home Assistant backup: STATUS=OK, archive includes configuration.yaml, storage dir and DB file, offhost SHA256/listing OK, restore not attempted in observed proof. +- Healthchecks restore: STATUS=OK, SQLite integrity OK and temp restore verify OK. +- BookStack restore: STATUS=OK, DB dump OK. +- Vikunja restore: STATUS=OK, SQLite integrity OK and temp restore verify OK. +- Kopia restore: STATUS=OK, synthetic snapshot and restore compare OK. +- Observability config backup: STATUS=OK, includes prometheus.yml, alertmanager.yml and health rules, offhost SHA256/listing OK. +- Ntfy is covered as part of observability/alerting stack but dedicated service backup signal was not confirmed in this catalog. +- MinIO is included in app-stacks-batch backup from 20260617; current dedicated timer was not confirmed. +- Homebox, Node-RED, MinIO, Mealie, Linkding, n8n and Actual Budget were part of app-stacks batch backup proof from 20260617. + +### External service backups +- NetBird VPS backup: STATUS=OK, snapshot under /mnt/staging/netbird-vps-backups/snapshots. +- NetBird VPS offhost: STATUS=OK to pve02. +- NetBird VPS restore validation: STATUS=OK, archive SHA256 recorded. +- CodeVipe/forum-prod: VM160 Proxmox backup is present; pve02 also has codevipe-cloud-backup.timer. +- XenForo SMTP rotation has DB backup proof from earlier rotation work; production forum application-level backup catalog should be expanded separately with CodeVipe-specific DB/files/cron/public cutover details. + +### Retention and cleanup +- Retention dry-run: STATUS=OK, production changed NO, destructive actions NO. +- Disk retention policy: STATUS=OK, root used pct observed 70 on edge-vm, removed dirs 0, Docker volume prune NO. +- App backup retention dry-run timer exists on edge-vm. +- homelab-backup-freshness timer exists on pve01. +- rollback-image-retention health file exists but had permission issue during one raw read; it must be read with sudo in future collection commands. + +### Known coverage gaps / backlog +- backup-restore-coverage-slo remains a coverage backlog, not a runtime blocker. +- Actual Budget latest proof says RESTORE_ATTEMPTED=NO. +- Home Assistant latest proof says RESTORE_ATTEMPTED=NO. +- Linkding latest proof says RESTORE_ATTEMPTED=NO. +- Karakeep latest proof says RESTORE_ATTEMPTED=NO. +- Mealie latest proof says RESTORE_ATTEMPTED=NO. +- n8n latest proof says RESTORE_ATTEMPTED=NO. +- Observability config latest proof says RESTORE_ATTEMPTED=NO. +- Router config backup is currently reference/self-test based, not confirmed as recurring automated backup. +- Syncthing, ntfy, MinIO and several P2 services need either dedicated backup/restore proof or explicit classification as disposable/rebuildable. + +### Proof +- Raw backup/offhost/restore inventory: 138_BACKUP_RESTORE_OFFHOST_CATALOG_RAW_INVENTORY.txt. +- Backup timer summary draft: 139_BACKUP_CATALOG_SERVICE_SUMMARY_DRAFT.txt. +- Normalized backup health index: 140_BACKUP_HEALTH_FILES_NORMALIZED_INDEX.txt. +## MONITORING_ALERTING_HEALTH_REFERENCE_20260630 + +### Monitoring stack +- Primary monitoring host: edge-vm, [PRIVATE_IP]. +- Prometheus container: prometheus, local port 127.0.0.1:9090. +- Grafana container: grafana, local port 127.0.0.1:3003, VPN route grafana.vpn.gram1.ru. +- Alertmanager container: alertmanager, local port 127.0.0.1:9093. +- Loki container: loki, local port 127.0.0.1:3100. +- Alloy container: alloy, local port 127.0.0.1:12345. +- Uptime Kuma container: uptime-kuma, local port 127.0.0.1:3001, public route uptime.gram1.ru and VPN route kuma.vpn.gram1.ru. +- Gotify container: gotify, local port 127.0.0.1:8082, public route gotify.gram1.ru. +- ntfy container: ntfy, local port 127.0.0.1:8055, VPN route ntfy.vpn.gram1.ru. +- Healthchecks container: healthchecks, local port 127.0.0.1:8015, VPN route checks.vpn.gram1.ru. +- Blackbox exporter: blackbox-exporter, local port 127.0.0.1:9115. +- cAdvisor: cadvisor, local port 127.0.0.1:8081. +- Node exporter on edge-vm: node-exporter, local port 127.0.0.1:9100. +- Beszel: beszel, local port 127.0.0.1:8090. +- Dozzle: dozzle, local port 127.0.0.1:9999. + +### PVE monitoring endpoints +- pve01: prometheus-node-exporter active on :9100, homelab-health-http active on :9101, Proxmox pveproxy on :8006. +- pve02: prometheus-node-exporter active on :9100, homelab-health-http active on :9101, Proxmox pveproxy on :8006. +- pve03: prometheus-node-exporter active on :9100, Proxmox pveproxy on :8006. +- pve03 has node-exporter/SMART timers but no homelab-health-http :9101 endpoint in observed output. +- PVE smartctl textfile timers exist on pve01/pve02/pve03. +- pve01 also runs private VPN host health, NetBird peer health, disk space health, MkDocs refresh, evidence catalog and quality gate timers. + +### Prometheus config +- Prometheus scrape interval: 30s. +- Prometheus evaluation interval: 30s. +- Rule files path: /etc/prometheus/rules/*.yml. +- Alertmanager target: alertmanager:9093. +- Blackbox HTTP job targets: + - https://nc.gram1.ru + - https://git.gram1.ru + - https://auth.gram1.ru + - https://paper.gram1.ru + - https://backup.gram1.ru +- Edge node exporter scrape target: node-exporter:9100. +- PVE node exporter targets: [PRIVATE_IP]:9100, [PRIVATE_IP]:9100, [PRIVATE_IP]:9100. +- Proxmox exporter job targets pve01/pve02/pve03 through exporter endpoint [PRIVATE_IP]:9221. +- cAdvisor scrape target: cadvisor:8080. + +### Alertmanager / notification routing +- Alertmanager route receiver: ntfy. +- Alertmanager webhook target: http://ntfy/homelab-alerts. +- Alert group_by: alertname, instance, severity. +- group_wait: 10s. +- group_interval: 5m. +- repeat_interval: 4h. +- Alert routing proof: alert-routing.txt STATUS=OK, RULES_ADDED=6, PROMETHEUS_RELOAD=OK, TEST_NOTIFICATION_SENT=YES, NTFY_MESSAGES_PUBLISHED_DELTA=1. +- alert-routing-health.txt is the standardized status alias and is STATUS=OK. + +### Core Prometheus alert rules +- HomelabPrometheusTargetDown: up == 0 for 5m, severity warning. +- HomelabSmartDiskHealthFailed: homelab_smartctl_health_passed == 0 for 10m, severity critical. +- HomelabRootFilesystemLowSpace: root filesystem available below 15% for 15m, severity warning. +- HomelabPrometheusConfigReloadFailed: prometheus_config_last_reload_successful == 0 for 5m, severity critical. +- HomelabP0HealthStatusNotOk: homelab_health_status tier P0 equals 0 for 15m. +- HomelabP0BackupRestoreHealthStale: P0 backup/offhost/restore health older than 36h. +- HomelabP0WeeklyEdgeVmVzdumpHealthStale: weekly edge-vm vzdump health older than 8d. +- HomelabP1BackupHealthStale: P1 backup health older than 7d. +- HomelabMailCloudRestoreDrillStale: mail-cloud restore drill older than 8d. +- HomelabMkdocsRuntimePublishStale: MkDocs runtime publish older than 30d. +- HomelabHealthMetricsCollectorMissing: homelab health metrics absent from Prometheus. +- HomelabExternalCanaryHealthNotOkOrStale: external canary not OK or older than 30m. +- HomelabBlackboxProbeFailed / Missing: blackbox probe failed or absent. +- HomelabVulnScanHealthNotOkOrStale: vulnerability scan failed or older than 8d. +- HomelabNpmplusCertExpiryHealthNotOkOrStale: NPMplus cert expiry health failed or older than 48h. +- HomelabBlackboxTlsCertificateExpiringSoon: TLS certificate expiry within 14 days. + +### Loki / Alloy log pipeline +- Loki version observed: grafana/loki:3.7.2. +- Alloy version observed: grafana/alloy:v1.17.0. +- Loki auth_enabled: false. +- Loki storage: local filesystem under /loki. +- Loki schema: tsdb v13. +- Loki retention_period: 14d. +- Alloy discovers Docker through socket-proxy at tcp://socket-proxy:2375. +- Alloy relabels container, compose_project, compose_service and host=edge-vm. +- Alloy forwards Docker logs to http://loki:3100/loki/api/v1/push. +- Loki readiness observed as "ready". + +### Runtime dashboard semantics +- backup-restore-dashboard.txt is an authoritative runtime dashboard file. +- Current observed runtime dashboard: STATUS=OK, ITEMS=69, NOT_OK=0. +- backup-restore-dashboard.json confirmed not_ok=[]. +- restore-drill-index.txt observed: STATUS=OK, RESTORE_PROOFS=16/16, MISSING empty. +- cluster-daily-status.txt observed: STATUS=OK, PROBLEM_COUNT=0. +- external-canary.txt observed: STATUS=OK, BAD=0. +- external canary checks include nc.gram1.ru, git.gram1.ru, auth.gram1.ru, backup.gram1.ru. +- Alertmanager local API is reachable. +- Gotify local /health returns green. +- ntfy local /v1/health returns healthy true. +- Uptime Kuma local endpoint returns HTTP 302 to /dashboard, which is expected for UI auth/redirect. +- Healthchecks local status endpoint returns HTTP 301, which is treated as reachable. + +### Certificate / vulnerability health +- npmplus-cert-expiry-health.txt is the standardized health alias for certificate expiry. +- npmplus-certificate-expiry.txt is the detailed cert expiry file. +- Detailed cert expiry file observed cert_count=9 and min_cert_days_left=75 with warn threshold 30. +- cluster-daily-status local extra check reports npmplus-cert-expiry status OK, problems=0, min_days=75. +- trivy-scan.txt observed: STATUS=OK, IMAGE_TOTAL=58, IMAGE_SCAN_OK=58, IMAGE_SCAN_ERROR=0, CRITICAL_TOTAL=112, HIGH_TOTAL=1650. +- vuln-scan.txt observed: STATUS=OK, MODE=full, IMAGES=58, TRIVY_OK=49, CRITICAL_MATCHES=203. + +### SLO backlog semantics +- Runtime dashboard OK does not mean SLO backlog is closed. +- slo-coverage-backlog-index.txt observed: STATUS=OK, BACKLOG_WARN_COUNT=12, RUNTIME_BLOCKER=NO. +- backup-restore-coverage-slo.txt observed: STATUS=WARN, APP_TOTAL=43, GREEN_COUNT=3, YELLOW_COUNT=21, RED_COUNT=19, MISSING_BACKUP_SIGNAL=31, MISSING_RESTORE_SIGNAL=28, MISSING_TIMER=31, MISSING_EXTERNAL_VALIDATION=43. +- Other observed WARN backlog files include container-image-lifecycle-slo, cron-job-monitoring-slo, direct-heartbeat-pilot-readiness, healthchecks-heartbeat-coverage, healthchecks-job-coverage, healthchecks-job-monitoring-slo, homepage-service-catalog-slo, ingress-dns-route-slo, observability-health-surface-slo and security-vulnerability-slo. +- SLO WARN files are tracked separately and are not runtime blockers unless explicitly promoted to P0 runtime checks. + +### Important file locations +- Main health dir on edge-vm: /var/lib/homelab-health. +- Prometheus config: /opt/stacks/observability-lite/prometheus/prometheus.yml. +- Prometheus rules: /opt/stacks/observability-lite/prometheus/rules/. +- Alertmanager config: /opt/stacks/observability-lite/alertmanager/alertmanager.yml. +- Loki config: /opt/stacks/loki-alloy/loki-config.yaml. +- Alloy config: /opt/stacks/loki-alloy/config.alloy. +- Blackbox exporter config: /opt/stacks/blackbox-exporter/blackbox.yml. +- Uptime Kuma DB: /opt/uptime-kuma-compose/data/kuma.db. +- Gotify DB: /opt/gotify-compose/data/gotify.db. +- ntfy DB/cache: /opt/stacks/ntfy/data/user.db and /opt/stacks/ntfy/cache/cache.db. +- Healthchecks settings/data: /opt/stacks/healthchecks/local_settings.py and /var/lib/healthchecks. + +### Known caveats +- A previous compact check looked for npmplus-cert-expiry.txt; use npmplus-certificate-expiry.txt for detailed cert expiry and npmplus-cert-expiry-health.txt for standardized health. +- health-metrics.txt is not required to exist as a health file; health metrics may be emitted via node-exporter textfile collector. +- Prometheus targets must be checked with explicit HTTP code and parsed JSON output, not a silent curl pipeline. + +### Proof +- Raw monitoring inventory: 143_MONITORING_ALERTING_HEALTH_RAW_INVENTORY.txt. +- Runtime/SLO/alerting status: 144_MONITORING_RUNTIME_SLO_ALERTING_STATUS.txt. +- Kuma/Gotify/ntfy/Healthchecks status: 145_KUMA_GOTIFY_NTFY_HEALTHCHECKS_SAFE_STATUS.txt. +- Corrected monitoring compact status: 146_MONITORING_COMPACT_STATUS_CORRECTED.txt. + +## PROMETHEUS_STATUS_CORRECTION_20260630 + +- Initial monitoring compact check after reference creation reported Prometheus API HTTP_CODE=000 on 127.0.0.1:9090. +- This was recorded as error-register item 30 and checked with proof files 149_PROMETHEUS_DIAG_AFTER_MONITORING_REFERENCE.txt, 150_PROMETHEUS_CONDITIONAL_RECOVERY.txt and 151_PROMETHEUS_REFERENCE_CORRECTION_PROOF.txt. +- The Monitoring/Alerting section must be interpreted together with the latest Prometheus correction proof, not only with the earlier secret-scan proof. + +## COMMAND_PREFLIGHT_RULE_20260630 + +- Strict operator rule: before every command, check both the error register and this reference file. +- Required visible markers before main action: ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK. +- If either check fails, do not run the main action. +- Do not use exit 1 in interactive SSH sessions. +- Proof: 152_COMMAND_PREFLIGHT_RULE_RECORDED.txt. + +## PROMETHEUS_TARGETS_SETTLED_20260630 + +- Prometheus was absent after the initial monitoring reference proof and was restarted in 150_PROMETHEUS_CONDITIONAL_RECOVERY.txt. +- Prometheus readiness proof exists in 151_PROMETHEUS_REFERENCE_CORRECTION_PROOF.txt. +- Target health was rechecked with an additional settled scrape proof in 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt. +- Interpret Prometheus status from the latest settled proof, not from the earlier immediate-after-start unknown target state. +## PROMETHEUS_TARGETS_SETTLED_PROOF_CORRECTION_20260630 + +- 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt is invalid and must not be used as evidence. +- Reason: nested Python inside SSH lost quoting and produced SyntaxError. +- Error-register item 32 records this mistake. +- Valid replacement proof: 154_PROMETHEUS_TARGETS_SETTLED_PROOF_NO_PYTHON.txt. +- Prometheus status must be interpreted from 154 and later proofs, not from 153. + +## MONITORING_PROMETHEUS_FINAL_CLOSED_20260630 + +- Prometheus was initially absent after monitoring reference creation. +- It was restarted and then verified after scrape settling. +- Valid settled proof: 154_PROMETHEUS_TARGETS_SETTLED_PROOF_NO_PYTHON.txt. +- Final observed state: HTTP_CODE=200, PROM_TARGETS_TOTAL=14, PROM_UP_COUNT=14, PROM_NON_UP_COUNT=0. +- Correction/secret scan proof: 156_PROMETHEUS_CORRECTION_PROOF_AND_SECRET_SCAN.txt. +- Invalid proof 153 must not be used. +## SECURITY_ACCESS_SECRETS_OPERATING_MODEL_20260630 + +### Command safety +- Before every command, check /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md and current 31_HOMELAB_REFERENCE.md. +- Print ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK before the main action. +- Do not run the main action if either check fails. +- Do not use exit 1 in interactive SSH sessions. +- Avoid long nested SSH/Python/PHP/SQL quoting chains. +- Do not print secrets. + +### Access model +- Primary operator entrypoint: root@pve01 / [PRIVATE_IP]. +- pve02 and pve03 are reached as root from pve01. +- edge-vm is reached as debian@[PRIVATE_IP] with sudo. +- forum-prod is reached through pve02 using [SENSITIVE_PATH] to root@[PRIVATE_IP]. +- PVE users observed: root@pam and prometheus@pve. +- prometheus@pve has PVEAuditor on / for Proxmox exporter access. + +### SSH and permissions +- pve01 root keys observed: id_rsa, id_ed25519 and public keys. +- pve02 root keys observed: id_rsa and forum-prod-ci-key. +- pve03 root key observed: id_rsa. +- edge-vm root key observed: id_ed25519; debian authorized_keys observed. +- Security finding: pve01/pve02/pve03 root authorized_keys files were observed as mode 777. +- Correction proof: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt. +- Private key material must never be copied into the reference. + +### Secret storage +- Primary private storage root: /var/lib/homelab-private. +- Edge private secrets root: /var/lib/homelab-private/secrets. +- NPMplus DB: /opt/npmplus/npmplus/database.sqlite, root-only mode observed. +- Gotify DB: /opt/gotify-compose/data/gotify.db. +- Uptime Kuma DB: /opt/uptime-kuma-compose/data/kuma.db. +- Vaultwarden DB: /opt/vaultwarden-compose/data/db.sqlite3. +- Rclone configs exist under root config paths and must not be printed. +- Env/secret inventory is path-only: owner, mode, size and path only. + +### Network exposure +- PVE nodes expose SSH :22, Proxmox :8006 and node-exporter :9100. +- pve01/pve02 expose homelab-health-http :9101. +- pve01 exposes NUT :3493 on localhost and [PRIVATE_IP]. +- edge-vm exposes public :80/:443 through NPMplus. +- edge-vm NPMplus admin :81 and socket-proxy :2375 are bound to 127.0.0.1. +- edge-vm registry cache :5000 is bound to [PRIVATE_IP]. +- edge-vm Home Assistant :8123 is intentionally LAN-exposed. +- Edge ingress hardening proof reports STATUS=OK. + +### Rotation and scan proofs +- Cloudflare token rotation completed without printing token values. +- Old Cloudflare token revocation was externally confirmed. +- XenForo SMTP password rotation completed without printing password values. +- XenForo SMTP auth was verified with STARTTLS-safe method. +- Current reference strict scan shows zero strict secret hits. +- Selected generated reference blocks show zero strict secret hits. + +### Proof +- Raw security/access inventory: 158_SECURITY_ACCESS_SECRETS_RAW_INVENTORY.txt. +- Secret scan and rotation proof index: 159_SECURITY_SECRET_SCAN_AND_ROTATION_PROOF_INDEX.txt. +- SSH permission correction proof: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt. +## SECURITY_SSH_PERMISSION_CORRECTION_20260630 + +- Initial proof 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt is not sufficient as closure evidence. +- Reason: stat without -L still showed 777 after chmod, which may be symlink mode rather than target file mode. +- Error-register item 35 records this proof-quality issue. +- Authoritative replacement proof: 163_PVE_ROOT_AUTHORIZED_KEYS_SYMLINK_AWARE_FIX.txt. +- Valid closure condition: target permissions checked with stat -L are 600 for authorized_keys files and [SENSITIVE_PATH] is 700. +- Final observed status in replacement proof: STATUS=SSH_AUTHORIZED_KEYS_TARGET_PERMISSIONS_OK. +## STORAGE_DISK_SMART_CAPACITY_REFERENCE_20260630 + +### Proxmox storage model +- Cluster storage is defined in /etc/pve/storage.cfg. +- local storage: dir /var/lib/vz, content iso,vztmpl,backup,import. +- local-lvm storage: lvmthin data, content rootdir,images. +- Nightly vzdump job writes to local storage. +- VM/CT images are primarily stored on local-lvm. +- Staging/offhost paths are under /mnt/staging where present. + +### Node disks and capacity +- pve01: primary disk previously inventoried as Lexar SSD NQ7A1 1TB. +- pve02: primary disk previously inventoried as SK800-1TB. +- pve03: primary disk previously inventoried as Samsung 870 EVO 500GB. +- pve01 storage usage was previously observed around local 32.56% and local-lvm 17.50%. +- pve02 storage usage was previously observed around local 12.86% and local-lvm 11.65%. +- pve03 storage usage was previously observed around local 35.79% and local-lvm 64.84%. +- Raw current df/lsblk/LVM/SMART evidence is in 166_STORAGE_DISK_SMART_CAPACITY_RAW_INVENTORY.txt. + +### Edge VM storage model +- edge-vm runs Docker application stacks. +- Docker app data lives mainly under /opt/stacks, /opt/npmplus, /opt/gotify-compose, /opt/uptime-kuma-compose, /opt/vaultwarden-compose and /var/lib application paths. +- Immich media is mounted separately at /mnt/immich-media in the container mapping. +- Docker volume and image usage is captured in docker system df output. +- Disk retention health is tracked by disk-retention-policy and disk-retention-cleanup files. + +### SMART and health monitoring +- PVE nodes run smartctl textfile timers. +- pve01/pve02/pve03 expose node-exporter :9100 for Prometheus. +- pve01 and pve02 expose homelab-health-http :9101. +- SMART/disk/capacity health evidence is collected from /var/lib/homelab-health where present. +- Prometheus alert rule HomelabSmartDiskHealthFailed watches homelab_smartctl_health_passed == 0. +- Prometheus alert rule HomelabRootFilesystemLowSpace watches root filesystem available below 15%. +- pve03 did not have /var/lib/homelab-health in previous backup-health collection, but it has node-exporter and smartctl timers. + +### Retention and cleanup +- Edge disk retention policy previously observed STATUS=OK. +- Edge disk retention policy observed ROOT_USED_PCT=70, REMOVED_DIRS=0, DOCKER_VOLUME_PRUNE=NO. +- Retention dry-run policy is designed to avoid destructive production changes. +- Broad Docker prune is not used as a default cleanup mechanism. +- Cleanup and retention actions must have explicit proof files. + +### Operational rules +- Before deleting or pruning storage, create or identify rollback/backup proof. +- Do not delete vzdump/offhost/cloud backup artifacts unless retention policy and proof explicitly allow it. +- Do not infer offhost success from a failed rsync. +- For backup artifacts, verify exact file, size, checksum or zstd/vma integrity where applicable. +- For SMART, use health output plus selected attributes, not full raw dumps with unnecessary noise. + +### Proof +- Raw storage/disk/SMART/capacity inventory: 166_STORAGE_DISK_SMART_CAPACITY_RAW_INVENTORY.txt. +- Earlier Proxmox node/storage inventory: 120_PROXMOX_CLUSTER_NODE_STORAGE_NETWORK_INVENTORY.txt. +- Backup/restore catalog: 141_REFERENCE_BACKUP_RESTORE_OFFHOST_CATALOG_BLOCK.txt. +- Monitoring rules and SMART alert model: 147_REFERENCE_MONITORING_ALERTING_HEALTH_BLOCK.txt. +## STORAGE_CAPACITY_CORRECTION_20260630 + +- Storage block integrity review proof: 169_STORAGE_BLOCK_INTEGRITY_AND_PVE03_CAPACITY_REVIEW.txt. +- 167_REFERENCE_STORAGE_DISK_SMART_CAPACITY_BLOCK.txt integrity status: OK. +- pve03 /mnt/staging current observed use percent: 77. +- pve03 staging is close to the existing WARN threshold of 80% if observed use is 75% or higher. +- pve01 disk-space.txt coverage visibly listed pve01_staging, pve02_staging and edge_immich_media; pve03_staging coverage must be added or explicitly documented elsewhere. +- Storage layer is operationally OK only if SMART remains PASSED and pve03 staging is tracked before it crosses WARN/CRIT thresholds. +- This is a capacity coverage note, not a secret or runtime outage. +## PVE03_STAGING_CAPACITY_MONITOR_20260630 + +- pve03 /mnt/staging was observed at 77% usage. +- Existing pve01 disk-space.txt did not visibly include pve03_staging coverage. +- A dedicated pve03 staging capacity health check was added on pve01. +- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt. +- Timer: homelab-pve03-staging-capacity-health.timer. +- Service: homelab-pve03-staging-capacity-health.service. +- Current observed status: OK. +- WARN threshold: 80%. +- CRIT threshold: 90%. +- Proof: 173_PVE03_STAGING_CAPACITY_HEALTH_INSTALLED.txt. + +## STORAGE_LAYER_FINAL_CLOSED_WITH_PVE03_MONITOR_20260630 + +- Storage/disk/SMART/capacity layer is closed with an explicit pve03 staging capacity monitor. +- pve03 /mnt/staging was near WARN threshold at 77%, so a dedicated health file and timer were added. +- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt. +- Timer: homelab-pve03-staging-capacity-health.timer. +- Final closure proof: 176_STORAGE_FINAL_CLOSURE_WITH_PVE03_MONITOR.txt. +## SERVICE_DEPENDENCY_MAP_20260630 + +### Ingress and DNS chain +- Home DHCP DNS: [PRIVATE_IP] and [PRIVATE_IP]. +- dns1: CT110 / [PRIVATE_IP] / AdGuard Home. +- dns2: CT111 / [PRIVATE_IP] / AdGuard Home. +- unbound1: CT112 / [PRIVATE_IP]:5335. +- unbound2: CT113 / [PRIVATE_IP]:5335. +- dns1 upstream: [PRIVATE_IP]:5335. +- dns2 upstream: [PRIVATE_IP]:5335. +- Public ingress HTTP/HTTPS: router forwards TCP 80/443 to edge-vm [PRIVATE_IP]. +- Reverse proxy: NPMplus on edge-vm, container npmplus, host networking, admin bound to 127.0.0.1:81. +- Edge runtime host: VM130 edge-vm, [PRIVATE_IP], Docker Compose projects count 43. +- NPMplus proxy routes count: 47. +- NPMplus certificates and proxy route source proof: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt. + +### Core public routes +| Route | Upstream | Runtime owner | Backup/health evidence | +|---|---|---|---| +| nc.gram1.ru | http://[PRIVATE_IP]:11000 | VM150 Nextcloud AIO | VM150 vzdump, nextcloud restore proof, external canary | +| uptime.gram1.ru | http://127.0.0.1:3001 | edge-vm / uptime-kuma | npmplus-kuma backup/restore, Kuma health | +| gotify.gram1.ru | http://127.0.0.1:8082 | edge-vm / gotify | gotify health, alert-routing proof | +| vault.gram1.ru | http://127.0.0.1:8083 | edge-vm / vaultwarden | vaultwarden backup/offhost/restore | +| dockge.gram1.ru | http://127.0.0.1:5001 | edge-vm / dockge | stack inventory | +| home.gram1.ru | http://127.0.0.1:3000 | edge-vm / homepage | homepage container, dashboard route | +| git.gram1.ru | http://127.0.0.1:3002 | edge-vm / gitea | gitea backup/offhost/restore | +| dozzle.gram1.ru | http://127.0.0.1:9999 | edge-vm / dozzle | docker stack inventory | +| paper.gram1.ru | http://127.0.0.1:8010 | edge-vm / paperless | paperless backup/offhost/restore | +| memos.gram1.ru | http://127.0.0.1:5230 | edge-vm / memos | memos backup/offhost/restore | +| photos.gram1.ru | http://127.0.0.1:2283 | edge-vm / immich | immich media/full consistency proof | +| auth.gram1.ru | http://127.0.0.1:9000 | edge-vm / authentik | authentik backup/offhost/restore | +| backup.gram1.ru | http://[PRIVATE_IP]:9101 | pve01 / homelab-health-http | backup dashboard and health proofs | + +### VPN routes on wildcard certificate +| Route | Upstream | Runtime owner | +|---|---|---| +| kuma.vpn.gram1.ru | http://127.0.0.1:3001 | uptime-kuma | +| homepage.vpn.gram1.ru | http://127.0.0.1:3000 | homepage | +| gitea.vpn.gram1.ru | http://127.0.0.1:3002 | gitea | +| filebrowser.vpn.gram1.ru | http://127.0.0.1:8085 | filebrowser | +| auth.vpn.gram1.ru | http://127.0.0.1:9000 | authentik | +| actual-budget.vpn.gram1.ru | http://127.0.0.1:5006 | actual-budget | +| grafana.vpn.gram1.ru | http://127.0.0.1:3003 | grafana | +| homebox.vpn.gram1.ru | http://127.0.0.1:7745 | homebox | +| mealie.vpn.gram1.ru | http://127.0.0.1:9001 | mealie | +| n8n.vpn.gram1.ru | http://127.0.0.1:5678 | n8n | +| netbox.vpn.gram1.ru | http://127.0.0.1:8000 | netbox | +| node-red.vpn.gram1.ru | http://127.0.0.1:1880 | node-red | +| beszel.vpn.gram1.ru | http://127.0.0.1:8090 | beszel | +| it-tools.vpn.gram1.ru | http://127.0.0.1:8084 | it-tools | +| karakeep.vpn.gram1.ru | http://127.0.0.1:3030 | karakeep | +| linkding.vpn.gram1.ru | http://127.0.0.1:9091 | linkding | +| minio.vpn.gram1.ru | http://127.0.0.1:9003 | minio | +| ntfy.vpn.gram1.ru | http://127.0.0.1:8055 | ntfy | +| searxng.vpn.gram1.ru | http://127.0.0.1:8888 | searxng | +| syncthing.vpn.gram1.ru | http://127.0.0.1:8384 | syncthing | +| alertmanager.vpn.gram1.ru | http://127.0.0.1:9093 | alertmanager | +| prometheus.vpn.gram1.ru | http://127.0.0.1:9090 | prometheus | +| homeassistant.vpn.gram1.ru | http://127.0.0.1:8123 | homeassistant | +| npmplus.vpn.gram1.ru | https://[PRIVATE_IP]:81 | NPMplus admin, localhost-bound on edge | +| proxmox.vpn.gram1.ru | https://[PRIVATE_IP]:8006 | pve01 Proxmox UI | +| dns1.vpn.gram1.ru | http://[PRIVATE_IP]:8080 | dns1 AdGuard | +| dns2.vpn.gram1.ru | http://[PRIVATE_IP]:8080 | dns2 AdGuard | +| checks.vpn.gram1.ru | http://127.0.0.1:8015 | healthchecks | +| vikunja.vpn.gram1.ru | http://127.0.0.1:8016 | vikunja | +| bookstack.vpn.gram1.ru | http://127.0.0.1:8017 | bookstack | +| pdf.vpn.gram1.ru | http://127.0.0.1:8018 | stirling-pdf | +| jellyfin.vpn.gram1.ru | http://127.0.0.1:8019 | jellyfin | +| audiobooks.vpn.gram1.ru | http://127.0.0.1:8020 | audiobookshelf | +| books.vpn.gram1.ru | http://127.0.0.1:8021 | calibre-web | + +### Disabled / special routes +- npm.gram1.ru exists in NPMplus but was observed disabled. +- turn.gram1.ru is an AdGuard rewrite to [PRIVATE_IP] for Nextcloud Talk/TURN. +- Router forwards TCP/UDP 3478 to Nextcloud [PRIVATE_IP]. +- Router forwards TCP/UDP 51820 for WireGuard on the Home bridge, not to edge-vm. + +### Critical dependency rules +- If a public app is down, check in this order: DNS rewrite/upstream, NPMplus route/cert, upstream container/VM, app health file, backup/restore proof. +- If a VPN route is down, check NetBird first, then NPMplus wildcard cert, then local upstream. +- If DNS is broken, check dns1/dns2 AdGuard and unbound1/unbound2 before app containers. +- If NPMplus is broken, use backup/restore evidence from npmplus-kuma-config and NPMplus DB backup. +- If edge-vm is broken, VM130 full-image backup/offhost/restore proofs are authoritative. +- If pve03 storage approaches WARN/CRIT, use pve03-staging-capacity health file. + +### Proof +- Raw service dependency evidence index: 177_SERVICE_DEPENDENCY_MAP_RAW_EVIDENCE_INDEX.txt. +- NPMplus route/cert rows: 129_NPMPLUS_PROXY_CERT_STREAM_ROWS_SAFE.txt. +- AdGuard/Unbound rewrites/upstreams: 130_ADGUARD_UPSTREAM_REWRITES_UNBOUND_RULES_SAFE.txt. +- Edge Docker container normalized map: 134_EDGE_DOCKER_CONTAINER_NORMALIZED_MAP.txt. +- Proxmox VM/CT inventory: 121_PROXMOX_VM_CT_CONFIG_INVENTORY_REDACTED.txt. +- Backup catalog: 141_REFERENCE_BACKUP_RESTORE_OFFHOST_CATALOG_BLOCK.txt. +- Monitoring model: 147_REFERENCE_MONITORING_ALERTING_HEALTH_BLOCK.txt. + +## SERVICE_DEPENDENCY_MAP_FINAL_CLOSED_20260630 + +- Service Dependency Map layer is closed. +- Integrity proof: 180_SERVICE_DEPENDENCY_MAP_INTEGRITY_REVIEW.txt. +- Secret scan proof: 179_SERVICE_DEPENDENCY_MAP_REFERENCE_PROOF_AND_SECRET_SCAN.txt. +- Final closure proof: 181_SERVICE_DEPENDENCY_MAP_FINAL_CLOSURE.txt. +## RUNBOOKS_RECOVERY_PROCEDURES_20260630 + +### Universal operator preflight +- Before every command, check /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md and the current 31_HOMELAB_REFERENCE.md. +- Required visible markers: ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK. +- If preflight fails, do not run the main action. +- Do not use exit 1 in interactive SSH sessions. +- Do not paste full terminal transcripts back into shell. +- Do not print secrets. +- Prefer short proof-producing commands over long nested quoting chains. + +### First triage order +- Check current reference layer first. +- Check latest proof file named by the relevant reference layer. +- Check health files under /var/lib/homelab-health where applicable. +- Check runtime state only after understanding the owning node, VM, container and proxy route. +- Record every failed command or misleading proof in the error register before moving on. + +### Public application down +- Start with Service Dependency Map. +- Check DNS path: dns1/dns2 AdGuard, then unbound1/unbound2. +- Check NPMplus route and certificate using NPMplus DB/proxy proof. +- Check upstream app container or VM. +- Check app-specific health, backup and restore proof. +- Check external-canary and blackbox/Prometheus if route is public. +- Do not change DNS, certificates or proxy routes without DB backup and proof. + +### DNS failure +- dns1 is CT110 at [PRIVATE_IP]. +- dns2 is CT111 at [PRIVATE_IP]. +- unbound1 is CT112 at [PRIVATE_IP]:5335. +- unbound2 is CT113 at [PRIVATE_IP]:5335. +- AdGuard upstreams must point to local Unbound pair. +- Rewrites include public routes for git, dozzle, paper, memos, photos, auth and backup to edge-vm. +- turn.gram1.ru points to Nextcloud [PRIVATE_IP]. +- Use DNS/Ingress reference and proof files before editing configs. + +### Ingress / NPMplus failure +- Edge VM is VM130 at [PRIVATE_IP]. +- NPMplus listens publicly on 80/443 and admin is bound to 127.0.0.1:81. +- NPMplus DB is /opt/npmplus/npmplus/database.sqlite. +- Before modifying certificates or proxy rows, create a DB backup. +- Cloudflare token values must never be printed. +- Use npmplus-kuma-config backup/offhost/restore proofs for recovery. + +### Edge VM failure +- VM130 is the Docker runtime host. +- First check Proxmox VM state and pve03 storage. +- Then use VM130 full-image vzdump/offhost/restore proofs. +- Current authoritative edge-vm backup/offhost/restore evidence is in backup catalog and VM130 proofs. +- If restoring, verify exact archive, checksum and VMA/ZSTD integrity before booting replacement. + +### Proxmox / VM / CT restore +- Primary nightly job: homelab-nightly-all at 03:30, zstd, snapshot. +- Included VMIDs: 110,111,112,113,130,150,160. +- Use backup catalog for latest known backup/offhost/restore evidence. +- Do not infer offhost success from failed rsync. +- Verify exact artifact, size and checksum where available. +- For VM130 and VM160, use their dedicated closure proofs. + +### Monitoring / alerting failure +- Prometheus is on edge-vm at 127.0.0.1:9090. +- Alertmanager is on 127.0.0.1:9093. +- Loki is on 127.0.0.1:3100. +- Uptime Kuma is on 127.0.0.1:3001. +- Gotify is on 127.0.0.1:8082. +- ntfy is on 127.0.0.1:8055. +- Healthchecks is on 127.0.0.1:8015. +- Prometheus proof 153 is invalid and must not be used. +- Use proof 154 and final closure 157 for Prometheus settled target state. + +### Backup dashboard / SLO interpretation +- Runtime dashboard OK means current operational checks are green. +- SLO backlog WARN means coverage improvement backlog, not necessarily runtime outage. +- Do not merge SLO backlog WARN into runtime blockers unless explicitly promoted. +- Use slo-coverage-backlog-index for backlog status. + +### Storage / capacity event +- pve03 staging is monitored separately because it was observed at 77%. +- Health file: /var/lib/homelab-health/pve03-staging-capacity.txt. +- WARN threshold: 80%. +- CRIT threshold: 90%. +- Timer: homelab-pve03-staging-capacity-health.timer. +- Before cleanup, check retention policy and backup/offhost proof. +- Do not use broad Docker prune by default. + +### Security / secret incident +- Stop printing values immediately. +- Identify whether the leak is value, file path, or false-positive text. +- Rotate affected token/password if a value was exposed. +- Create backup before DB/config mutation. +- Re-run strict secret scan after rotation. +- Record proof files and external revocation confirmation where applicable. +- Cloudflare token and XenForo SMTP rotations already have closure proofs. + +### Forum / CodeVipe incident +- forum-prod is VM160 at [PRIVATE_IP]. +- Access path is through pve02 using [SENSITIVE_PATH] +- XenForo path: /var/www/codevipe/public. +- SMTP host: mail.pvepro.ru on port 587 with STARTTLS semantics. +- Do not enable smtpSsl=true blindly for port 587. +- Do not use fragile nested PHP/base64 SMTP checkers. +- Use XenForo SMTP rotation and auth proof files for current mail state. + +### Final evidence rules +- Every remediation gets a numbered proof file. +- Every reference block gets a proof and secret scan file. +- Invalid proof files must be explicitly superseded, not silently ignored. +- Reference closure requires integrity scan, secret scan and required-heading checks. +## HOMELAB_REFERENCE_FINAL_OPERATOR_STATUS_20260630 + +### Status +- Current homelab reference is complete for the audited local infrastructure scope. +- Final quality precheck passed. +- Required headings are present. +- Bad terminal/log marker scan is clean. +- Strict secret scan is clean. +- This is a reference/operator-status closure, not an archive/export. + +### Closed layers +- Critical runtime tails closure. +- Proxmox cluster, node, storage and VM/CT inventory. +- DNS, ingress, NPMplus certificates and AdGuard/Unbound model. +- Edge Docker stacks and container map. +- Backup, restore, offhost and cloud backup catalog. +- Monitoring, alerting, health dashboard and Prometheus correction. +- Security, access and secrets operating model. +- Storage, disk, SMART and capacity model with pve03 staging monitor. +- Service Dependency Map. +- Runbooks and recovery procedures. + +### Important superseded/invalid proofs +- 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt is invalid and must not be used. +- 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED.txt is not sufficient by itself because symlink mode showed 777. +- Use 154 for Prometheus settled targets and 163 for symlink-aware SSH target permissions. + +### Current authoritative final proof set +- 185_HOMELAB_REFERENCE_FINAL_QUALITY_SCAN_PRECHECK.txt. +- 184_RUNBOOKS_RECOVERY_REFERENCE_PROOF_AND_SECRET_SCAN.txt. +- 181_SERVICE_DEPENDENCY_MAP_FINAL_CLOSURE.txt. +- 176_STORAGE_FINAL_CLOSURE_WITH_PVE03_MONITOR.txt. +- 165_SECURITY_ACCESS_SECRETS_FINAL_PROOF_AND_SECRET_SCAN.txt. +- 157_MONITORING_PROMETHEUS_FINAL_CLOSURE.txt. +- 142_BACKUP_RESTORE_OFFHOST_REFERENCE_PROOF_AND_SECRET_SCAN.txt. +- 136_EDGE_DOCKER_STACKS_REFERENCE_PROOF_AND_SECRET_SCAN.txt. +- 132_DNS_INGRESS_REFERENCE_PROOF_AND_SECRET_SCAN.txt. + +### Operator rule +- Before every future command, continue checking both the error register and this reference file. +- Required visible markers remain ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK. +## HOMELAB_REFERENCE_DEEP_AUDIT_CLOSURE_20260630 + +### Deep audit result +- Error register and reference consistency audit passed. +- Final proof files unresolved-marker audit passed. +- Reference proof-link audit passed. +- All referenced proof files exist. +- No unresolved final REVIEW_REQUIRED, PROOF_MISSING, HEADING_MISSING, BAD_MARKER, STRICT_SECRET, RULE_CHECK_FAIL, SyntaxError or PERM_BAD markers remained in final proof set. +- Invalid/superseded proofs are explicitly documented and must not be used as authoritative closure evidence. + +### Authoritative deep audit proofs +- 189_DEEP_ERROR_REGISTER_REFERENCE_CONSISTENCY_AUDIT.txt. +- 190_DEEP_PROOF_FILES_UNRESOLVED_MARKER_AUDIT.txt. +- 191_DEEP_REFERENCE_PROOF_LINK_AUDIT.txt. + +### Scope statement +- This closes the current audited local homelab reference scope. +- External systems can still receive separate dedicated passports if the scope is expanded later. + +## NEXTCLOUD_VM150_MAIL_CLOUD_BACKUP_20260630 +- VM150 Nextcloud Mail-cloud backup is installed on pve01. +- Timer: homelab-mail-cloud-nextcloud-vm-upload.timer. +- Health file: /var/lib/homelab-health/mail-cloud-nextcloud-vm.txt. +- Current proof health: STATUS=OK, MODE=recurring-chunked, PARTS=14, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no. +- Proof: 192_NEXTCLOUD_VM150_MAIL_CLOUD_CHUNKED_PROOF.txt. + +## ROUTER_RECURRING_BACKUP_BLOCKER_20260630 +- Router recurring backup from pve01 is not installed yet. +- Reason: pve01 cannot reach router management ports 2222, 5080, 5083 or 2323 on [PRIVATE_IP]. +- Current router backup state remains reference/self-test based until router management access is allowed or startup-config is exported by another approved path. +- Do not run bash/POSIX automation against Netcraze CLI; use router CLI syntax only. +- Proof: 195_ROUTER_RECURRING_BACKUP_BLOCKED_BY_MGMT_ACL_PROOF.txt. + +## EDGE_GAP_RESTORE_DRYRUN_CLOSURE_20260630 +- Restore dry-run gaps closed for Actual Budget, Home Assistant, Karakeep, Linkding, Mealie, n8n and Observability config. +- Production data was not restored or modified; checks used temporary extraction, SHA verification, tar listing/extraction and SQLite integrity where applicable. +- Current observed restore health: all 7 STATUS=OK with RESTORE_ATTEMPTED=YES and SECRET_PRINTED=no. +- Proof: 197_EDGE_GAP_RESTORE_DRYRUN_CLOSURE_PROOF.txt. + +## P2_SMALL_STACKS_BACKUP_RESTORE_CLOSURE_20260630 +- P2 small-stack backup and restore dry-run is installed on edge-vm. +- Covered paths: Syncthing config, ntfy cache/data, MinIO data, Homebox data and Node-RED data. +- Timer: homelab-p2-small-stacks-backup-restore.timer. +- Current observed health: STATUS=OK, BACKUP_PATH_COUNT=6, RESTORE_ATTEMPTED=YES, SQLITE_INTEGRITY_OK=yes, SECRET_PRINTED=no. +- Proof: 199_P2_SMALL_STACKS_BACKUP_RESTORE_CLOSURE_PROOF.txt. + +## BACKUP_CLOUD_AND_RESTORE_FINAL_CLOSURE_20260630 +- Backup/cloud/restore improvement pass is closed with runtime dashboard STATUS=OK and NOT_OK=0. +- VM150 Nextcloud now has Mail-cloud chunked backup with download verification. +- Restore dry-run gaps closed for Actual Budget, Home Assistant, Karakeep, Linkding, Mealie, n8n and Observability config. +- P2 small stacks now have backup and restore dry-run coverage for Syncthing config, ntfy, MinIO, Homebox and Node-RED. +- Router recurring backup remains blocked by management ACL and is documented as blocker, not silently closed. +- Proof: 201_BACKUP_CLOUD_AND_RESTORE_FINAL_CLOSURE_PROOF.txt. + +## ROUTER_STARTUP_CONFIG_MAIL_CLOUD_MANUAL_20260630 +- Router startup-config manual backup is stored in Mail-cloud crypt remote. +- Source file content is not printed in proofs or reference. +- Health file: /var/lib/homelab-health/router-startup-config-mail-cloud.txt. +- Current observed health: STATUS=OK, UPLOAD_OK=yes, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no. +- Recurring router backup is still blocked until pve01 can reach router management ports. +- Proof: 203_ROUTER_STARTUP_CONFIG_MAIL_CLOUD_MANUAL_PROOF.txt. + +## ROUTER_RUNNING_CONFIG_MAIL_CLOUD_RECURRING_20260630 +- Router running-config recurring Mail-cloud backup is installed on pve01. +- Timer: homelab-router-running-config-mail-cloud.timer. +- Health file: /var/lib/homelab-health/router-running-config-mail-cloud.txt. +- Current observed health: STATUS=OK, SSH_RC=0, LOOKS_CONFIG=yes, UPLOAD_OK=yes, DOWNLOAD_VERIFY=yes, SECRET_PRINTED=no. +- Manual startup-config Mail-cloud backup also exists as separate proof. +- Proof: 205_ROUTER_RUNNING_CONFIG_MAIL_CLOUD_RECURRING_PROOF.txt. + +## POST_BACKUP_PASS_SLO_RECONCILIATION_20260630 +- Post backup/cloud/restore pass SLO reconciliation is closed. +- Runtime backup dashboard remains STATUS=OK with NOT_OK=0. +- Restore index remains STATUS=OK with RESTORE_PROOFS=16/16. +- VM150 Mail-cloud backup, router manual/recurring Mail-cloud backups, P2 small-stacks and 7 edge restore dry-runs are all confirmed OK. +- Existing backup-restore-coverage-slo.txt is historical model output from 20260625 and does not represent current runtime blocker state. +- Current reconciliation health: STATUS=OK, RUNTIME_BLOCKER=NO, SECRET_PRINTED=no. +- Proof: 208_POST_BACKUP_PASS_SLO_RECONCILIATION_PROOF.txt. + +## POST_BACKUP_PASS_ALERTING_20260630 +- Prometheus alerting for post-backup-pass health files is installed and loaded. +- Rule file: /opt/stacks/observability-lite/prometheus/rules/homelab-post-backup-pass-alerts.yml. +- Alerts: HomelabPostBackupPassHealthNotOk and HomelabPostBackupPassHealthStale. +- Covered checks include VM150 Mail-cloud, router startup/running config Mail-cloud, P2 small-stacks and seven restore dry-run health files. +- Current proof confirms rule validation, Prometheus API visibility and up targets. +- Proof: 210_POST_BACKUP_PASS_ALERTING_PROOF.txt. + +## POST_BACKUP_PASS_ALERTS_NOT_FIRING_20260630 +- Post-backup-pass Prometheus alert rules are loaded and currently not firing. +- Corrected proof uses direct PromQL ALERTS query for firing state, not grep over rule names. +- Alerts checked: HomelabPostBackupPassHealthNotOk and HomelabPostBackupPassHealthStale. +- Proof: 213_POST_BACKUP_PASS_ALERTS_NOT_FIRING_CORRECTED_PROOF.txt. + +## MAIL_CLOUD_CAPACITY_RETENTION_20260630 +- Mail-cloud capacity and backup directory inventory was checked after post-backup-pass closure. +- VM150 Nextcloud, router startup/running config, P0 critical and edge-vm backup directories are present on crypt remotes. +- Observed capacity is sufficient: largest used remote was pve03-mail-03-crypt around 65 GiB of 1 TiB. +- Retention shape observed: P0 critical has 10 visible dirs with keep 14, edge-vm has 3 visible dirs with keep 4, new VM150/router jobs have initial dirs. +- Proof: 216_MAIL_CLOUD_CAPACITY_RETENTION_PROOF.txt. + +## ROUTER_BACKUP_SECRET_PERMISSION_20260630 +- Router recurring backup uses a dedicated routerbackup credential file on pve01. +- Secret file path: /var/lib/homelab-private/secrets/routerbackup.pass. +- File content must never be printed; only mode/owner/size may be checked. +- Current observed permission target: root-owned mode 600. +- Proof: 218_ROUTER_BACKUP_SECRET_PERMISSION_PROOF.txt. + +## NEW_BACKUP_UNITS_SYSTEMD_INVENTORY_20260630 +- New backup/restore systemd units and timers were inventoried after post-backup-pass closure. +- pve01 units checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup. +- edge-vm units checked: P2 small-stacks backup/restore, backup dashboard and restore drill index. +- Proof records LoadState, ActiveState, UnitFileState and failed-unit counts for pve01 and edge-vm. +- Proof: 222_NEW_BACKUP_UNITS_SYSTEMD_INVENTORY_PROOF.txt. + +## POST_BACKUP_PASS_PROM_HEALTH_COVERAGE_20260630 +- Prometheus health coverage was checked for the post-backup-pass checks. +- Covered checks: VM150 Mail-cloud, router startup/running config, P2 small-stacks and seven restore dry-runs. +- Proof records health status and age from Prometheus without printing credential values. +- Proof: 224_POST_BACKUP_PASS_PROM_HEALTH_COVERAGE_PROOF.txt. + +## POST_BACKUP_PASS_AUDIT_INDEX_20260630 +- Audit proof manifest was generated for post-backup-pass evidence files 192-225. +- Manifest records file names, sha256 hashes, count and missing-number check. +- Proof: 226_POST_BACKUP_PASS_AUDIT_INDEX_PROOF.txt. + +## POST_BACKUP_PASS_EXTENDED_FINAL_SNAPSHOT_20260630 +- Extended post-backup-pass final snapshot was created after capacity, retention, unit inventory, Prometheus coverage and audit index checks. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-227. +- Proof: 228_POST_BACKUP_PASS_EXTENDED_FINAL_SNAPSHOT_PROOF.txt. + +## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_229_20260630 +- Final audit manifest was generated after the extended final snapshot. +- Manifest covers proof numbers 192-229, including snapshot and reference-scan proofs. +- Manifest records file names, sha256 hashes, count and missing-number check. +- Proof: 230_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_229_PROOF.txt. + +## NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_20260630 +- New backup/restore unit files and executable script paths were inventoried and hashed. +- pve01 services checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup. +- edge-vm services checked: P2 small-stacks backup/restore, backup dashboard and restore drill index. +- Proof records FragmentPath, ExecStart path, mode/owner/size and sha256 hashes without printing script contents. +- Proof: 232_NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_PROOF.txt. + +## NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_CORRECTED_20260630 +- Corrected integrity proof was generated because one edge-vm executable required sudo for sha256sum. +- Corrected proof records hashes for pve01 and edge-vm unit fragments and executable scripts without printing script contents. +- Proof: 234_NEW_BACKUP_UNITS_AND_SCRIPTS_INTEGRITY_CORRECTED_PROOF.txt. + +## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_235_20260630 +- Final audit manifest was regenerated after corrected unit/script integrity proof. +- Manifest covers proof numbers 192-235, including corrected integrity proof 234/235. +- Manifest records file names, sha256 hashes, count and missing-number check. +- Proof: 236_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_235_PROOF.txt. + +## NEW_BACKUP_TIMERS_INTEGRITY_20260630 +- New backup/restore timer unit files were inventoried and hashed. +- pve01 timers checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup. +- edge-vm timers checked: P2 small-stacks backup/restore, backup dashboard and restore drill index. +- Proof records FragmentPath, mode/owner/size, sha256 hashes and next-elapse metadata without printing sensitive values. +- Proof: 238_NEW_BACKUP_TIMERS_INTEGRITY_PROOF.txt. + +## POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_239_20260630 +- Final audit manifest was regenerated after timer integrity proof. +- Manifest covers proof numbers 192-239 and records file names, sha256 hashes, count and missing-number check. +- Proof: 240_POST_BACKUP_PASS_FINAL_AUDIT_INDEX_192_239_PROOF.txt. + +## POST_BACKUP_PASS_FINAL_SNAPSHOT_AFTER_TIMERS_20260630 +- Final snapshot was created after timer integrity and final audit manifest 192-239. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-241. +- Proof: 242_POST_BACKUP_PASS_FINAL_SNAPSHOT_AFTER_TIMERS_PROOF.txt. + +## ASSISTANT_COMMAND_BATCHING_RULE_20260630 +- Operator preference: combine related homelab commands into fewer package-style runs to reduce slow repetitive manual work. +- Applies to proof generation, reference updates, validation scans, hash/index refreshes and snapshot steps when safe. +- Mandatory preflight and sensitive-output hygiene still take priority. + +## POST_BACKUP_PASS_AUDIT_INDEX_AFTER_BATCHING_RULE_192_245_20260630 +- Audit manifest was regenerated after assistant command batching rule was recorded. +- Manifest covers proof numbers 192-245 and records file names, sha256 hashes, count and missing-number check. +- Proof: 246_POST_BACKUP_PASS_AUDIT_INDEX_AFTER_BATCHING_RULE_192_245_PROOF.txt. + +## POST_BACKUP_PASS_SNAPSHOT_AFTER_BATCHING_RULE_20260630 +- Snapshot was created after assistant command batching rule and audit manifest 192-245. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-247. +- Proof: 248_POST_BACKUP_PASS_SNAPSHOT_AFTER_BATCHING_RULE_PROOF.txt. + +## ALERTMANAGER_ROUTING_AND_CONFIG_20260630 +- Alertmanager routing/config health was checked after post-backup-pass alert rules were installed. +- Proof records container presence, config validation, readiness endpoint and sanitized route/receiver metadata. +- Sensitive receiver values and URLs are intentionally not printed. +- Proof: 250_ALERTMANAGER_ROUTING_AND_CONFIG_PROOF.txt. + +## AUDIT_INDEX_192_251_20260630 +- Audit manifest was regenerated after Alertmanager routing/config proof. +- Manifest covers proof numbers 192-251 with count and missing-number check. +- Proof: 252_AUDIT_INDEX_192_251_PROOF.txt. + +## RCLONE_CONFIG_PERMISSION_AND_REMOTE_INVENTORY_20260630 +- rclone config permissions and crypt remote inventory were checked without printing config contents. +- Proof records config owner/mode/size, remote names and capacity summaries for active Mail-cloud crypt remotes. +- Proof: 254_RCLONE_CONFIG_PERMISSION_AND_REMOTE_INVENTORY_PROOF.txt. + +## AUDIT_INDEX_192_255_20260630 +- Audit manifest was regenerated after rclone config/remote inventory proof. +- Manifest covers proof numbers 192-255 with count and missing-number check. +- Proof: 256_AUDIT_INDEX_192_255_PROOF.txt. + +## CURRENT_OPERATIONAL_ROLLUP_20260630 +- Current operational rollup was captured after rclone config/remote inventory proof. +- Rollup records failed-unit counts, next timers, Mail-cloud capacity, Prometheus target count, post-backup health status/age, post-backup firing alerts and Alertmanager readiness. +- Proof: 258_CURRENT_OPERATIONAL_ROLLUP_PROOF.txt. + +## AUDIT_INDEX_192_259_20260630 +- Audit manifest was regenerated after current operational rollup. +- Manifest covers proof numbers 192-259 with count and missing-number check. +- Proof: 260_AUDIT_INDEX_192_259_PROOF.txt. + +## SNAPSHOT_AFTER_CURRENT_OPERATIONAL_ROLLUP_20260630 +- Snapshot was created after current operational rollup and audit index 192-259. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-261. +- Proof: 262_SNAPSHOT_AFTER_CURRENT_OPERATIONAL_ROLLUP_PROOF.txt. + +## AUDIT_INDEX_192_263_20260630 +- Audit manifest was regenerated after snapshot following current operational rollup. +- Manifest covers proof numbers 192-263 with count and missing-number check. +- Proof: 264_AUDIT_INDEX_192_263_PROOF.txt. + +## NEW_BACKUP_SERVICE_RUNTIME_RESULT_20260630 +- Runtime result metadata was captured for new backup/restore service units. +- pve01 services checked: VM150 Mail-cloud upload and router running-config Mail-cloud backup. +- edge-vm services checked: P2 small-stacks backup/restore, backup dashboard and restore drill index. +- Proof records LoadState, ActiveState, Result, ExecMainStatus and restart counters without printing sensitive values. +- Proof: 266_NEW_BACKUP_SERVICE_RUNTIME_RESULT_PROOF.txt. + +## AUDIT_INDEX_192_267_20260630 +- Audit manifest was regenerated after new backup service runtime-result proof. +- Manifest covers proof numbers 192-267 with count and missing-number check. +- Proof: 268_AUDIT_INDEX_192_267_PROOF.txt. + +## NEW_BACKUP_UNITS_JOURNAL_ERROR_SCAN_20260630 +- Journal warning/error counters were captured for new backup/restore units and timers over the last 24 hours. +- Proof records only counts, not journal message bodies, to avoid sensitive-output risk. +- Proof: 270_NEW_BACKUP_UNITS_JOURNAL_ERROR_SCAN_PROOF.txt. + +## AUDIT_INDEX_192_271_20260630 +- Audit manifest was regenerated after journal error scan proof. +- Manifest covers proof numbers 192-271 with count and missing-number check. +- Proof: 272_AUDIT_INDEX_192_271_PROOF.txt. + +## NEW_BACKUP_UNITS_JOURNAL_ERROR_TRIAGE_20260630 +- Journal warning/error triage was captured after warning/error counters were non-zero. +- Proof records per-unit warning/error counts and redacted journal fragments. +- Proof: 274_NEW_BACKUP_UNITS_JOURNAL_ERROR_TRIAGE_PROOF.txt. + +## AUDIT_INDEX_192_275_20260630 +- Audit manifest was regenerated after journal error triage proof. +- Manifest covers proof numbers 192-275 with count and missing-number check. +- Proof: 276_AUDIT_INDEX_192_275_PROOF.txt. + +## VM150_MAIL_CLOUD_JOURNAL_NOISE_CLASSIFICATION_20260630 +- VM150 Mail-cloud upload journal warnings/errors were classified after journal triage found non-zero counters. +- Current service result, Prometheus health and firing-alert status are used to decide whether journal noise is blocking. +- Proof includes redacted journal indicators only, not secrets. +- Proof: 278_VM150_MAIL_CLOUD_JOURNAL_NOISE_CLASSIFICATION_PROOF.txt. + +## AUDIT_INDEX_192_279_20260630 +- Audit manifest was regenerated after VM150 Mail-cloud journal-noise classification. +- Manifest covers proof numbers 192-279 with count and missing-number check. +- Proof: 280_AUDIT_INDEX_192_279_PROOF.txt. + +## SNAPSHOT_AFTER_VM150_JOURNAL_NOISE_CLASSIFICATION_20260630 +- Snapshot was created after VM150 Mail-cloud journal noise was classified as non-blocking historical noise. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-281. +- Proof: 282_SNAPSHOT_AFTER_VM150_JOURNAL_NOISE_CLASSIFICATION_PROOF.txt. + +## AUDIT_INDEX_192_283_20260630 +- Audit manifest was regenerated after snapshot following VM150 journal-noise classification. +- Manifest covers proof numbers 192-283 with count and missing-number check. +- Proof: 284_AUDIT_INDEX_192_283_PROOF.txt. + +## POST_BACKUP_PASS_CLOSURE_SUMMARY_20260630 +- Post-backup-pass closure summary was generated after VM150 journal-noise classification. +- Summary checks backup SLO closure, alert rules, firing-alert status, Prometheus health coverage, failed-unit state, Alertmanager readiness, rclone config hygiene, journal-noise classification and audit index completeness. +- Proof: 286_POST_BACKUP_PASS_CLOSURE_SUMMARY_PROOF.txt. + +## SNAPSHOT_AFTER_POST_BACKUP_PASS_CLOSURE_SUMMARY_20260630 +- Snapshot was created after post-backup-pass closure summary. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range: 192-287. +- Proof: 288_SNAPSHOT_AFTER_POST_BACKUP_PASS_CLOSURE_SUMMARY_PROOF.txt. + +## AUDIT_INDEX_192_289_20260630 +- Audit manifest was regenerated after post-backup-pass closure summary snapshot. +- Manifest covers proof numbers 192-289 with count and missing-number check. +- Proof: 290_AUDIT_INDEX_192_289_PROOF.txt. + +## TOMORROW_FIRST_RUN_VERIFICATION_PLAN_20260630 +- First scheduled-run verification should be done after 2026-07-01 08:15 MSK. +- Verify P2 small-stacks, router running-config Mail-cloud and VM150 Nextcloud Mail-cloud first automatic runs. +- Also verify service results, health status/age, Mail-cloud directories, dashboard, restore index, Prometheus firing alerts and Alertmanager readiness. +- Proof: 292_TOMORROW_FIRST_RUN_VERIFICATION_PLAN_PROOF.txt. + +## AUDIT_INDEX_192_293_20260630 +- Audit manifest was regenerated after tomorrow first-run verification plan. +- Manifest covers proof numbers 192-293 with count and missing-number check. +- Proof: 294_AUDIT_INDEX_192_293_PROOF.txt. + +## END_OF_DAY_POST_BACKUP_PASS_SNAPSHOT_20260630 +- End-of-day snapshot was created after post-backup-pass closure summary and tomorrow first-run verification plan. +- Snapshot copies current reference and assistant error register into the audit directory and records sha256 hashes. +- Covered proof range before this snapshot: 192-295. +- Proof: 296_END_OF_DAY_POST_BACKUP_PASS_SNAPSHOT_PROOF.txt. + +## AUDIT_INDEX_192_297_20260630 +- Final end-of-day audit manifest was generated after post-backup-pass snapshot. +- Manifest covers proof numbers 192-297 with count and missing-number check. +- Proof: 298_AUDIT_INDEX_192_297_PROOF.txt. + +## HOME_PORTAL_DISCOVERY_20260630 +- Home portal discovery started for https://home.gram1.ru/. +- Goal: make the portal show and open all web services, including external service cards such as NetBird and mail. +- Discovery proof records DNS, HTTPS/TLS headers, reverse-proxy candidates, compose files, domain references and homepage config candidates without printing secrets. +- Proof: 300_HOME_PORTAL_DISCOVERY_PROOF.txt. + +## HOME_PORTAL_CONFIG_AND_SERVICE_INVENTORY_20260630 +- Home portal config and service inventory was collected for https://home.gram1.ru/. +- Inventory records portal containers, config files, redacted card lines, compose web hints, domain references and common subdomain probes. +- Proof: 302_HOME_PORTAL_CONFIG_AND_SERVICE_INVENTORY_PROOF.txt. + +## HOME_PORTAL_GAP_ANALYSIS_20260630 +- Home portal gap analysis was collected for gethomepage/homepage behind npmplus. +- Proof records Homepage config hashes, redacted current cards/bookmarks, current URLs, npmplus route lines, local web-port probes and known service container candidates. +- Proof: 304_HOME_PORTAL_GAP_ANALYSIS_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_305_20260630 +- Home portal audit manifest was generated for proof numbers 300-305. +- Proof: 306_HOME_PORTAL_AUDIT_INDEX_300_305_PROOF.txt. + +## HOME_PORTAL_CARD_AND_API_ERROR_ANALYSIS_20260630 +- Home portal card/API-error analysis was collected before editing Homepage. +- Proof records current card URLs/titles, redacted widget config, redacted Homepage log errors, NPMPlus internal route files/routes and candidate public/VPN cards. +- Proof: 308_HOME_PORTAL_CARD_AND_API_ERROR_ANALYSIS_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_309_20260630 +- Home portal audit manifest was regenerated after card/API-error analysis. +- Manifest covers proof numbers 300-309 with count and missing-number check. +- Proof: 310_HOME_PORTAL_AUDIT_INDEX_300_309_PROOF.txt. + +## HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_APPLY_20260630 +- Homepage services.yaml was backed up, duplicate VPN cards were removed where a non-VPN card existed, and missing web cards for NetBird, Mail and Webmail were added without VPN suffix in the card names. +- SMTP/IMAP were intentionally not added as cards because they are protocol endpoints, not web UI cards. +- Homepage container was restarted and portal/card URLs were checked. +- Proof: 312_HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_APPLY_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_313_20260630 +- Home portal audit manifest was regenerated after duplicate-card cleanup and external-card addition. +- Manifest covers proof numbers 300-313 with count and missing-number check. +- Proof: 314_HOME_PORTAL_AUDIT_INDEX_300_313_PROOF.txt. + +## HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_CORRECTED_APPLY_20260630 +- Corrected Homepage apply removed duplicate VPN cards where a non-VPN card existed and added NetBird, Mail and Webmail cards with non-VPN names. +- The added card URLs use currently reachable vpn.gram1.ru web endpoints because public netbird/mail/webmail hosts did not resolve/open during analysis. +- SMTP/IMAP were not added as cards because they are protocol endpoints, not web UIs. +- Proof: 316_HOME_PORTAL_DEDUP_AND_EXTERNAL_CARDS_CORRECTED_APPLY_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_317_20260630 +- Home portal audit manifest was regenerated after corrected duplicate-card cleanup and external-card addition. +- Manifest covers proof numbers 300-317 with count and missing-number check. +- Proof: 318_HOME_PORTAL_AUDIT_INDEX_300_317_PROOF.txt. + +## HOMELAB_COMMAND_SAFETY_HARDENING_20260630 +- Command safety rule strengthened after the home portal base64 apply failure. +- Opaque generated payloads are not acceptable for homelab changes; commands must be transparent, bounded, and validated before modification. +- Success requires content-specific checks in addition to service/runtime checks. +- Proof: 320_HOMELAB_COMMAND_SAFETY_HARDENING_RULE_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_321_20260630 +- Home portal audit manifest was regenerated after command-safety hardening rule. +- Manifest covers proof numbers 300-321 with count and missing-number check. +- Proof: 322_HOME_PORTAL_AUDIT_INDEX_300_321_PROOF.txt. + +## HOME_PORTAL_POST_APPLY_VALIDATION_AND_API_TRIAGE_20260630 +- Home portal was validated after duplicate cleanup and external-card addition. +- Proof checks portal HTTP status, Homepage container state, required cards, duplicate VPN card pairs, current card URLs, redacted API-error logs and widget configuration. +- Proof: 324_HOME_PORTAL_POST_APPLY_VALIDATION_AND_API_TRIAGE_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_325_20260630 +- Home portal audit manifest was regenerated after post-apply validation and API triage. +- Manifest covers proof numbers 300-325 with count and missing-number check. +- Proof: 326_HOME_PORTAL_AUDIT_INDEX_300_325_PROOF.txt. + +## HOME_PORTAL_API_ERROR_ROOT_CAUSE_20260630 +- Homepage API error root-cause analysis was collected after the UI showed API errors. +- Current evidence points to the Open-Meteo/weather widget timing out from Homepage, separate from service cards. +- HTTP 200 for the portal is not sufficient for UI closure when Homepage logs still contain API errors. +- Proof: 328_HOME_PORTAL_API_ERROR_ROOT_CAUSE_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_329_20260630 +- Home portal audit manifest was regenerated after API-error root-cause analysis. +- Manifest covers proof numbers 300-329 with count and missing-number check. +- Proof: 330_HOME_PORTAL_AUDIT_INDEX_300_329_PROOF.txt. + +## HOME_PORTAL_DISABLE_OPENMETEO_WIDGET_20260630 +- Open-Meteo/weather widget was disabled after root-cause analysis showed Homepage API errors from api.open-meteo.com timeouts. +- This keeps the portal focused on service-launcher cards and prevents external weather API failures from causing UI API errors. +- Portal closure now requires current Homepage API-error logs to be zero after restart/reload. +- Proof: 332_HOME_PORTAL_DISABLE_OPENMETEO_WIDGET_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_333_20260630 +- Home portal audit manifest was regenerated after disabling Open-Meteo/weather widget. +- Manifest covers proof numbers 300-333 with count and missing-number check. +- Proof: 334_HOME_PORTAL_AUDIT_INDEX_300_333_PROOF.txt. + +## HOME_PORTAL_LINK_OPEN_AUDIT_20260630 +- Home portal card links were audited after Open-Meteo/weather widget was disabled. +- Audit extracts current Homepage href URLs and checks whether each opens with an acceptable HTTP status. +- Acceptable statuses include success, redirects and auth-gated 401/403 pages because those still indicate a reachable web service. +- Proof: 336_HOME_PORTAL_LINK_OPEN_AUDIT_PROOF.txt. + +## HOME_PORTAL_AUDIT_INDEX_300_337_20260630 +- Home portal audit manifest was regenerated after link-open audit. +- Manifest covers proof numbers 300-337 with count and missing-number check. +- Proof: 338_HOME_PORTAL_AUDIT_INDEX_300_337_PROOF.txt. + +## HOME_PORTAL_EXPLICIT_VPN_DUPLICATE_CLEANUP_CORRECTED_20260630 +- Explicit VPN duplicate cards were removed from Homepage with a short transparent perl edit: Homepage VPN, Authentik VPN, Gitea VPN and Uptime Kuma VPN. +- Old router URL [PRIVATE_IP] was replaced with [PRIVATE_IP] where present. +- Proof: 342_HOME_PORTAL_EXPLICIT_VPN_DUPLICATE_CLEANUP_CORRECTED_PROOF.txt. + +## HOME_PORTAL_REMAINING_DUPLICATE_OCCURRENCE_ANALYSIS_20260630 +- Remaining Homepage duplicate/VPN/router occurrences were inspected after corrected cleanup still reported REVIEW. +- Proof records exact redacted file/line occurrences before another edit. +- Proof: 344_HOME_PORTAL_REMAINING_DUPLICATE_OCCURRENCE_ANALYSIS_PROOF.txt. + +## HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RECHECK_20260630 +- Homepage duplicate cleanup was rechecked against active config files only, excluding logs and backup files. +- Previous broad grep included historical logs/backups and therefore overstated remaining active duplicates. +- Proof: 346_HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RECHECK_PROOF.txt. + +## HOME_PORTAL_ACTIVE_VPN_DUPLICATES_FINAL_CLEANUP_20260630 +- Active Homepage services/bookmarks files were cleaned from semantic VPN duplicate cards and old router URL references. +- Removed targets: Homepage VPN, Authentik VPN, Gitea VPN and Uptime Kuma VPN, plus matching vpn.gram1.ru duplicate URLs. +- Proof: 350_HOME_PORTAL_ACTIVE_VPN_DUPLICATES_FINAL_CLEANUP_PROOF.txt. + +## HOME_PORTAL_NPMPLUS_DEFAULT_ROUTE_ANALYSIS_20260630 +- Home portal NetBird/Mail/Webmail cards were investigated after user saw the NPMPlus default page from client [PRIVATE_IP]. +- Analysis checks URL body fingerprints, NPMPlus route config matches, service container candidates and Homepage card lines. +- Proof: 356_HOME_PORTAL_NPMPLUS_DEFAULT_ROUTE_ANALYSIS_PROOF.txt. + +## HOME_PORTAL_CURRENT_STATE_20260630 +- Active Homepage URL: https://home.gram1.ru. +- Homepage container is running and portal HTTP check returned 200 after latest changes. +- NetBird card points to https://nb.pvepro.ru. +- Mail card points to https://mail.pvepro.ru. +- Webmail card was removed; no separate Webmail card is currently configured. +- Router bookmark points to http://[PRIVATE_IP]:5080 for operator client LAN [PRIVATE_IP]. +- Homepage services.yaml was updated via YAML-aware edit for siteMonitor fields. +- Current active services.yaml has SITEMONITOR_COUNT=43. +- Latest validation before this reference update: YAML_ERRORS_LAST_5M=0, HOME_HTTP_CODE=200, HOMEPAGE_RUNNING=true. +- Do not use sed-only insertion for siteMonitor; use YAML-aware editing and validate YAML before restart. +- Related active-state checks: NB_LINK_ACTIVE=yes, MAIL_LINK_ACTIVE=yes, WEBMAIL_ACTIVE=no, ROUTER_192_168_1_1_ACTIVE=yes. + +## HOME_PORTAL_DIRECT_LINK_FIX_20260630 +- Direct Homepage link correction requested by operator. +- NetBird: https://nb.pvepro.ru. +- Mail: https://mail.pvepro.ru. +- Webmail card removed. +- Router restored to http://[PRIVATE_IP]:5080. + +## HOME_PORTAL_YAML_AWARE_SITEMONITOR_20260630 +- siteMonitor fields were added using YAML-aware PyYAML editing, not line-based sed insertion. +- Added siteMonitor to 43 service cards. +- Excluded cards: Homepage, NPMplus, Router and Public Domain. +- YAML validation passed before restart: YAML_VALIDATE_RC=0. +- Homepage restarted successfully and reported YAML_ERRORS=0. + +## HOMELAB_CLUSTER_BACKLOG_20260630 +- Cluster backlog was created at /etc/pve/HOMELAB_CLUSTER_BACKLOG.md. +- PBS is intentionally out of scope; backup strategy remains Mail-cloud based. +- First next action: finish Homepage final validation. + +## HOMEPAGE_BACKUP_GAP_CLASSIFICATION_20260630 +- Homepage backup coverage gap classification file created at /etc/pve/HOMEPAGE_BACKUP_GAP_CLASSIFICATION.md. +- It defines closure requirements for the 13 remaining Homepage backup coverage gap/special-review rows. +- PBS is intentionally out of scope; closure must use Mail-cloud backups or explicit stateless/config-only classification. + +## HOMEPAGE_EXISTING_BACKUP_EVIDENCE_CLOSURE_20260630 +- Homepage backup coverage matrix started closing existing-evidence rows. +- Router maps to existing Mail-cloud startup/running-config backup proofs when ROUTER_EXISTING_EVIDENCE=yes. +- MinIO Console maps to existing P2 small-stacks backup/restore proof when MINIO_EXISTING_EVIDENCE=yes. +- Proof: 364_HOMEPAGE_EXISTING_BACKUP_EVIDENCE_CLOSURE_PROOF.txt. + +## HOME_EXTERNAL_CLOUDFLARE_ALEXHOST_20260630 +- Homepage External group includes Cloudflare card: https://dash.cloudflare.com. +- Homepage External group includes AlexHost Billing card: https://bill.alexhost.com/. +- Both cards have siteMonitor enabled for green status dots. +- YAML validation passed, Homepage returned HTTP 200, container running, and no Homepage YAML errors were seen after restart. +- Proof: 368_HOME_EXTERNAL_CLOUDFLARE_ALEXHOST_PROOF.txt. + +## HOME_EXTERNAL_RELAY_REMOVED_20260630 +- Homepage External card/link for relay.pvepro.ru was removed because relay.pvepro.ru had DNS but no reachable HTTP/HTTPS endpoint from edge-vm. +- Proof: 382_HOME_EXTERNAL_RELAY_REMOVE_PROOF.txt. + +## ROUTER_CLI_PERMISSION_LIMIT_20260630 +- Netcraze routerbackup SSH access is read-only for backup: show running-config works, but ACL/config commands are denied. +- Homepage Moscow Router red-dot fix requires router ACL/Web UI/admin-level change, not routerbackup. +- Relevant proofs: 391, 393, 394, 395. + +## ROUTER_CLI_PROOF_REPAIR_20260630 +- Error register updated for blank proof summary extraction in 395. +- Corrected derived counts are recorded in 396_ROUTER_CLI_ERROR_REGISTER_REPAIR_PROOF.txt. + +## HOME_EXTERNAL_NETBIRD_ALEXHOST_NORMALIZE_20260630 +- Homepage External NetBird and AlexHost Billing href/siteMonitor URLs normalized with trailing slash. +- Homepage container node checks returned HTTP 200 for both URLs. +- Proof: 397_HOME_EXTERNAL_NETBIRD_ALEXHOST_NORMALIZE_PROOF.txt. + +## ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701 +- Moscow Router ACL is restored and correct after manual Web UI edits. +- Remaining blocker for Homepage green dot is HTTP/HTTPS 403 from router web service to edge-vm/Bridge1, not network reachability. +- Relevant proofs: 399, 400, 401, 402. + +## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_20260701 +- Moscow Router Homepage card href: http://[PRIVATE_IP]:5080. +- Moscow Router Homepage siteMonitor: http://[PRIVATE_IP]:18083/router-moscow. +- Health endpoint service on edge-vm: homelab-router-moscow-health-http.service. +- Endpoint semantics: returns 200 when TCP to router [PRIVATE_IP]:5080 succeeds, else 503. +- Relevant proofs: 405, 406, 407. + +## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_FIXED_20260701 +- Moscow Router Homepage card target after fixed apply: href http://[PRIVATE_IP]:5080, siteMonitor http://[PRIVATE_IP]:18083/router-moscow. +- Fixed updater handles Homepage service-name-key YAML shape and property-style YAML shape. +- Relevant proofs: 405, 406, 407, 408. + +## HOME_MOSCOW_ROUTER_HEALTH_ENDPOINT_APPLIED_EXACT_20260701 +- Moscow Router Homepage card exact active YAML shape was service-key style. +- Applied href: http://[PRIVATE_IP]:5080. +- Applied siteMonitor: http://[PRIVATE_IP]:18083/router-moscow. +- Health endpoint service: homelab-router-moscow-health-http.service on edge-vm. +- Relevant proofs: 406, 409, 410. + +## FORUM_PROD_VM160_REBUILD_BASELINE_20260701 +- Old test VM160 was stopped and destroyed cleanly: proof 419, CONFIG_EXISTS=no, VM160_DISKS_LEFT=0. +- New VM160 forum-prod was recreated on pve02 with 4 vCPU, 8 GiB RAM, 200G disk, MAC BC:24:11:B6:45:ED, IP [PRIVATE_IP]/24, gateway [PRIVATE_IP]. +- New VM identity confirmed: forum-prod / forum-prod.gram1.ru, Debian 12 bookworm, SSH OK, qemu-agent OK, chrony OK. +- Swapfile 8G enabled with /swapfile mode 600 and fstab entry: proof 430. Proof 429 is invalid due nested command substitution quoting. +- SURY PHP repo added; PHP 8.5.7 available and installed with Nginx 1.22.1, MariaDB 10.11.14 and required XenForo modules. +- Beget-compatible profile applied: memory_limit 256M, post/upload 1024M, max_input_vars 10000, MariaDB utf8mb4/utf8mb4_unicode_ci, innodb_buffer_pool_size 2G. +- Operator reference copies placed in /root/homelab-operator inside VM160 for local VM command preflight. + +## FORUM_PROD_CODEVIPE_FIRST_RECIPE_STRATEGY_20260701 +- Current VM160 is now laboratory state, not final production closure. +- Bulk import of five forums reached DB/files/nginx/php-fpm ready state, but frontend body stayed empty under PHP 8.5.7. +- Strategy changed: prove CodeVipe first on controlled runtime, likely PHP 8.3 or PHP 8.2, then destroy VM160 and rebuild all five forums from scratch using the proven recipe. +- Do not publish Cloudflare/NPMplus routes for these forums until CodeVipe recipe is proven and final rebuild is complete. +- Proofs: failed/diagnostic chain 451-480; next snapshot proof 481 before further lab work. + +## FORUM_PROD_FRESH5_DEPLOYED_OK_20260701 +- VM160 forum-prod on pve02, IP [PRIVATE_IP]. +- Runtime: Debian 12, Nginx, MariaDB 10.11, PHP 8.3-FPM. +- Five XenForo forums deployed from fresh ZIP backups: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- Webroots: /var/www/forums/{codevipe,gamevipe,hkmods,zakrutim,dsmods}/public. +- DBs: xf_codevipe, xf_gamevipe, xf_hkmods, xf_zakrutim, xf_dsmods. +- Final health proof inside VM: /root/evidence/513_FORUM_PROD_FRESH5_FINAL_HEALTH_PROOF.txt. +- Final result: all five frontend/admin HTTP 200, www redirects 301, internal_data 403, no new XenForo errors. +- Lesson: fresh file ZIPs contain populated internal_data/code_cache; broken branch lacked compiled template cache. + +## OPERATOR_RULE_CONTINUE_WITH_NEXT_COMMAND_20260701 +- Rule: when an infrastructure step ends at an expected prompt or safe checkpoint, do not stop with only a status summary. +- Rule: immediately provide the next executable command in the same response. +- Applies to: SSH enter/exit points, VM prompt confirmations, snapshot confirmations, file copy confirmations, successful health checks, and other obvious next-step transitions. +- Exception: stop only if the previous output contains an error, ambiguity, dangerous state, missing file, secret exposure risk, or a required user decision. + +## OPERATOR_RULE_BATCH_2_3_COMMANDS_20260701 +- Rule: when safe, provide 2-3 sequential executable commands in one response instead of stopping after each expected checkpoint. +- Applies after: successful prompt transitions, clean health checks, completed copies, completed snapshots, and non-destructive inventory. +- Do not batch commands across: destructive operations, public exposure, DNS/SSL changes, secret handling, unclear errors, missing files, or required user decisions. +- Each command must still print ERROR_REGISTER_CHECK=OK and REFERENCE_CHECK=OK before work. + +## FORUM_PROD_FRESH5_PUBLIC_CUTOVER_OK_20260701 +- Public cutover completed for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- Cloudflare A records for main and www names point to edge public IP 95.84.154.183. +- Edge NPMplus manual routes terminate TLS with real Let’s Encrypt certificates and proxy to forum-prod VM160 at [PRIVATE_IP]:80. +- Final public proof: /root/evidence/546_PUBLIC_FORUMS_FINAL_DNS_HTTPS_PROOF.txt on pve01. +- Expected public state: main domains HTTPS 200 with ssl_verify_result 0; HTTP redirects to HTTPS; www redirects to main domain through forum-prod canonical vhosts. + +## EDGE_FORUM_PUBLICATION_BACKUP_20260701 +- Edge NPMplus forum publication backup created after public cutover. +- Backup location: /opt/npmplus/manual-backups/forum-public-ok-*.tar.gz on edge-vm [PRIVATE_IP]. +- Includes manual proxy_host configs 200-204 and Let’s Encrypt forum certificates under /opt/npmplus/tls/forum-certs. + +## FORUM_CERT_RENEWAL_CONFIGURED_20260701 +- Edge certificate renewal configured on edge-vm [PRIVATE_IP]. +- Token stored at /opt/npmplus/secure/forum_cf_token.env mode 600 root-only. +- Renewal script: /opt/npmplus/scripts/renew-forum-certs.sh. +- Cron: /etc/cron.d/forum-cert-renew, daily 03:17. +- Proof: /root/evidence/550_EDGE_FORUM_CERT_RENEWAL_SETUP_PROOF.txt on pve01. + +## CLOUDFLARE_FORUM_DNS_AUDIT_OK_20260701 +- Full Cloudflare DNS audit completed for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- Proof: /root/evidence/554_CLOUDFLARE_FULL_DNS_AUDIT_PROOF.txt on pve01. +- Result: root and www A records for all five zones point to 95.84.154.183. +- Result: no root/www AAAA records, no root/www CNAME conflicts, no records with old IP 87.236.18.45. +- Remaining cdn.* CNAME and TXT/DMARC/DKIM records were observed and not changed. + +## FORUMS_FINAL_DNS_AUDIT_SNAPSHOT_OK_20260701 +- Final post-DNS-audit snapshot completed after shortening Proxmox snapshot name. +- Snapshot name: forum-dns-ok-065203Z +- Proof: /root/evidence/557_FINAL_SHORT_SNAPSHOT_AFTER_DNS_AUDIT_PROOF.txt on pve01. +- Current public state: all five main domains resolve to 95.84.154.183 and return HTTPS 200 with valid certificates. + +## FORUM_LOCAL_BACKUP_CONFIGURED_20260701 +- Local forum backup configured inside VM160 forum-prod. +- Script: /root/scripts/forum-backup.sh. +- Cron: /etc/cron.d/forum-local-backup, daily 02:42. +- Retention: removes per-run directories older than 14 days; compressed archives remain under /var/backups/forums. +- Proof: /root/evidence/560_FORUM_BACKUP_SCRIPT_SETUP_PROOF.txt on pve01. + +## FORUM_MAIL_NOREPLY_AND_SPF_CLEANUP_20260701 +- XenForo email fields defaultEmailAddress/contactEmailAddress/bounceEmailAddress/emailSenderName set to noreply@pvepro.ru for all five forums. +- SPF TXT on codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru changed from stale 87.236.18.45 to v=spf1 -all. +- DKIM, DMARC and cdn.* CNAME records were not changed. +- Proofs: 564_UPDATE_XENFORO_EMAIL_NOREPLY_PROOF.txt, 565_CF_UPDATE_FORUM_SPF_NOREPLY_PROOF.txt, 566_FORUM_MAIL_NOREPLY_FINAL_AUDIT_PROOF.txt. + +## FORUM_CDN_RECORDS_DELETED_20260701 +- Deleted cdn.* CNAME records for codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- Reason: old cdn.* records pointed to previous provider/trbcdn and no current public/local forum references were observed. +- Root and www A records remain on 95.84.154.183. +- Proofs: /root/evidence/567_CF_DELETE_FORUM_CDN_RECORDS_PROOF.txt and /root/evidence/568_CDN_DELETE_FINAL_AUDIT_REFERENCE_PROOF.txt on pve01. + +## FORUM_SPF_DEDUPED_AND_CDN_DELETED_20260701 +- Fixed duplicate SPF state caused by earlier failed cleanup attempt. +- Each of codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru now has exactly one SPF TXT: v=spf1 -all. +- Old SPF references to 87.236.18.* are absent. +- cdn.* CNAME records are absent. +- Public HTTPS remained healthy for all five forums. +- Proofs: 569_CF_FIX_DUPLICATE_SPF_RECORDS_PROOF.txt and 570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt. + +## FORUM_PUBLIC_DNS_MAIL_CLEAN_FINAL_20260701 +- Public web and mail-related DNS cleanup completed for five forum domains. +- @ and www A records point to 95.84.154.183. +- cdn.* CNAME records removed. +- SPF deduped to exactly one TXT: v=spf1 -all on each forum domain. +- Old provider IP 87.236.18.* absent from forum domain TXT records. +- XenForo visible email identity changed to noreply@pvepro.ru on all forums. +- Proofs: 570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt, 571_FORUM_MAIL_TRANSPORT_READINESS_PROOF.txt, 572_FORUM_PUBLIC_AND_MAIL_DNS_FINAL_SUMMARY_PROOF.txt. + +## FORUM_SMTP_TRANSPORT_PAUSED_20260701 +- Forum XenForo email identity is noreply@pvepro.ru, but actual SMTP transport is paused. +- Reason: failed SMTP authentication triggered Mailcow netfilter ban for 95.84.154.183. +- Broken msmtp secret/config files were removed from forum-prod. +- Do not re-enable SMTP until the mailbox/app password is rotated and tested with a single controlled attempt. +- Proof: /root/evidence/590_FORUM_SMTP_FAILED_CONFIG_DISABLED_PROOF.txt. + +## POST_INCIDENT_STABLE_STATE_20260701 +- Incident after failed forum SMTP testing resolved. +- Mailcow and NetBird are reachable again: mail.pvepro.ru/admin and nb.pvepro.ru return HTTP 200. +- Mailcow netfilter ban for 95.84.154.183 was removed on VPS mail.pvepro.ru. +- Forum sites remain healthy over HTTPS. +- Forum SMTP transport remains paused; broken msmtp configs/secrets removed from forum-prod. +- Do not retry SMTP until the mailbox/app password is rotated. +- Proofs: 589_POST_UNBAN_MAILCOW_NETBIRD_VERIFY.txt on VPS, 590_FORUM_SMTP_FAILED_CONFIG_DISABLED_PROOF.txt, 591_POST_INCIDENT_PUBLIC_STATUS_PROOF.txt, 592_POST_INCIDENT_FINAL_STABLE_STATE_PROOF.txt. + +## FORUM_SMTP_ONE_SHOT_TEST_OK_20260701 +- One controlled SMTP test from forum-prod to mail.pvepro.ru:587 succeeded using rotated credentials. +- No persistent forum SMTP config was enabled. +- One-shot secret/config files were removed after the test. +- Mailcow and NetBird remained reachable after the test. +- Proofs: /root/evidence/594_FORUM_SMTP_ONE_SHOT_TEST_PROOF.txt and /root/evidence/595_SMTP_TEST_AFTERCHECK_PROOF.txt. + +## SMTP_ONESHOT_OK_STABLE_FINAL_20260701 +- Rotated SMTP credential was tested once from forum-prod to mail.pvepro.ru:587 successfully. +- No persistent SMTP config or secret was left on forum-prod after the one-shot test. +- Mailcow and NetBird remained reachable after the test. +- Forum websites remained healthy. +- Persistent forum SMTP is still not enabled; enable it only with an explicit command and hidden password input. +- Proofs: 594_FORUM_SMTP_ONE_SHOT_TEST_PROOF.txt, 595_SMTP_TEST_AFTERCHECK_PROOF.txt, 596_SMTP_ONESHOT_OK_FINAL_STABLE_PROOF.txt. + +## FORUM_PERSISTENT_MSMTP_ENABLED_20260701 +- Persistent msmtp-mta transport enabled on forum-prod as /usr/sbin/sendmail. +- SMTP relay: mail.pvepro.ru:587 STARTTLS, sender noreply@pvepro.ru. +- Secret stored only as root:www-data 0640 base64 file under /etc/msmtp/pvepro.env; no raw SMTP_PASS file remains. +- PHP mail() as www-data succeeded after persistent config installation. +- Mailcow and NetBird remained reachable after enabling persistent forum SMTP. +- Proofs: /root/evidence/597_ENABLE_FORUM_PERSISTENT_MSMTP_PROOF.txt and /root/evidence/598_PERSISTENT_MSMTP_FINAL_VERIFY_PROOF.txt. + +## FORUM_PERSISTENT_SMTP_FINAL_STABLE_20260701 +- Persistent forum SMTP via msmtp is enabled on forum-prod. +- PHP mail() as www-data succeeded after installation. +- Forum domains remain healthy over HTTPS. +- Mailcow and NetBird remain reachable after enabling persistent forum SMTP. +- Secret storage is root:www-data 0640 under /etc/msmtp/pvepro.env; raw [SENSITIVE_PATH] SMTP_PASS file is absent. +- Snapshot after enable: forum-smtp-on-080840Z. +- Proofs: 597_ENABLE_FORUM_PERSISTENT_MSMTP_PROOF.txt, 598_PERSISTENT_MSMTP_FINAL_VERIFY_PROOF.txt, 600_FORUM_PERSISTENT_SMTP_FINAL_STABLE_PROOF.txt. + +## FORUM_PROD_FRESH5_MAILRU_CLOUD_BACKUP_OK_20260701 +- Existing Mail.ru Cloud rclone crypt remotes on pve02 were reused; no new cloud account was configured. +- Old codevipe-cloud-backup.timer was stale after fresh five-forum rebuild and is disabled. +- New pve02 timer forum-prod-fresh5-cloud-backup.timer uploads current forum-prod local archives from VM160 /var/backups/forums. +- Current backup contains all five forums: codevipe, gamevipe, hkmods, zakrutim, dsmods. +- Upload target: pve02-mail-01/02/03/04-crypt under app/forums/forum-prod/fresh5/. +- Archive is split into 1500M parts before upload, with SHA256SUMS and MANIFEST.txt. +- First fresh5 cloud upload proof: /root/evidence/608_FORUM_PROD_FRESH5_CLOUD_BACKUP_FIRST_RUN_PROOF.txt. + +## FORUM_PROD_FRESH5_MAILRU_CLOUD_RESTORE_DRILL_OK_20260701 +- Restore drill from Mail.ru Cloud crypt remote completed for forum-prod fresh5 backup. +- Remote source used: pve02-mail-01-crypt app/forums/forum-prod/fresh5 latest timestamp. +- Downloaded split parts, MANIFEST.txt, SHA256SUMS and SHA256SUMS.local were verified. +- Part SHA256 verification passed. +- Split archive was reconstructed and full archive hash matched SHA256SUMS.local. +- tar listing confirmed DB dumps and file archives for codevipe, gamevipe, hkmods, zakrutim and dsmods. +- Proof: /root/evidence/612_FORUM_FRESH5_CLOUD_RESTORE_DRILL_CORRECTED_PROOF.txt. + +## FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_OK_20260701 +- Five public XenForo forums are healthy over HTTPS. +- Mailcow admin and NetBird UI are reachable after persistent forum SMTP enablement. +- Persistent forum SMTP via msmtp is enabled and tested. +- Local backup exists on VM160 under /var/backups/forums. +- Mail.ru Cloud fresh5 backup is enabled on pve02 via forum-prod-fresh5-cloud-backup.timer. +- Old CodeVipe-only cloud timer is disabled. +- Cloud restore drill passed: split parts verified, archive reconstructed, SHA256 matched, tar contained DB/file archives for all five forums. +- Final rollup proof: /root/evidence/614_FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_PROOF.txt. + +## FORUM_XENFORO_JOB_RUNNER_CONFIGURED_20260701 +- XenForo CLI job runner configured inside VM160 forum-prod. +- Script: /root/scripts/forum-xenforo-run-jobs.sh. +- Cron: /etc/cron.d/forum-xenforo-run-jobs every 5 minutes. +- Purpose: process XenForo job queues and cron tasks independent of forum traffic. +- First run proof: /root/evidence/617_XENFORO_RUN_JOBS_ONCE_PROOF.txt. +- Cron verification proof: /root/evidence/618_XENFORO_JOB_RUNNER_CRON_CONFIG_PROOF.txt. + +## FORUM_XENFORO_CRON_DAEMON_AND_JOBRUNNER_OK_20260701 +- cron daemon inside VM160 forum-prod is active/enabled and was proven by temporary /etc/cron.d execution proof. +- XenForo job runner script executed successfully after cron daemon verification. +- Due XenForo cron count returned to zero after run. +- Proof: /root/evidence/621_FORUM_CRON_DAEMON_JOBRUNNER_VERIFY_PROOF.txt. + +## FORUM_DBTECH_TOR_TIMEOUT_CLASSIFIED_20260701 +- Each forum had one historical XenForo error_log row for cURL timeout to check.torproject.org/torbulkexitlist. +- Source addon: DBTech/Security DragonByte Security 5.0.0. +- dbtech_security_tornodes option is 0 on all five forums. +- Classified as non-blocking historical external network timeout/noise; not a forum runtime failure. +- Error rows were not deleted. +- Proof: /root/evidence/622_TOR_TIMEOUT_CLASSIFICATION_FINAL_PROOF.txt. + +## FORUM_XENFORO_BUILTIN_MAIL_DELIVERED_BUT_SPAM_20260701 +- Built-in XenForo outgoing email test from site "Моды для Hollow Knight" was delivered to aleisaev@yandex.ru. +- Sender was noreply@pvepro.ru. +- Yandex placed the message in Spam with warning that links/images were disabled. +- This proves forum SMTP transport works, but deliverability/reputation needs tuning. +- Custom proof 623 is invalid due /root path permission issue and is superseded by this user-observed built-in test. +- Next evidence needed: Yandex raw message headers, especially Authentication-Results and DKIM-Signature. +- Proof: /root/evidence/627_XENFORO_BUILTIN_MAIL_DELIVERED_SPAM_CLASSIFICATION_PROOF.txt. + +## FORUM_YANDEX_DELIVERABILITY_SPAM_WITH_SPF_DKIM_PASS_20260701 +- Built-in XenForo outgoing email test was delivered to aleisaev@yandex.ru but placed in Spam. +- User-provided Yandex headers showed SPF pass and DKIM pass for pvepro.ru. +- Return-Path, From and DKIM domain aligned on pvepro.ru. +- Yandex spam score observed: X-Yandex-Spam: 4. +- Classification: SMTP/Mailcow/XenForo transport is OK; remaining issue is deliverability/reputation/content filtering. +- DNS change is not required based on this header proof. +- Next actions: mark as Not spam in Yandex, add/check pvepro.ru in Yandex Postmaster, warm up sender reputation, optionally plan per-forum sender domains later. + +## FORUM_PROJECT_FINAL_ACCEPTANCE_OK_20260701 +- Five XenForo forums are published and healthy over HTTPS: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- VM160 forum-prod is the active production forum VM on pve02 at [PRIVATE_IP]. +- Runtime services are configured: nginx, MariaDB, PHP 8.3 FPM, qemu-agent, cron, msmtp/sendmail transport. +- NPMplus on edge terminates TLS and proxies all five forums to VM160. +- Let’s Encrypt certificates are active for all five domains. +- Forum SMTP transport works through noreply@pvepro.ru; Yandex headers proved SPF pass and DKIM pass. Spam placement is classified as reputation/content deliverability, not server failure. +- XenForo job runner cron is configured and cron daemon execution was proven. +- Local backup is configured on VM160 and Mail.ru Cloud fresh5/all-forums backup is configured on pve02. +- Cloud restore drill passed: split parts verified, archive SHA matched, tar contained DB/file/config archives for all five forums. +- DBTech Tor timeout rows are classified as historical external timeout noise and were not deleted. +- Final proof: /root/evidence/629_FORUM_PROJECT_FINAL_ACCEPTANCE_PROOF.txt. + +## FORUM_NEXT_CHAT_HANDOFF_FULL_STATE_20260701 + +### Start point +- Start shell: root@pve01. +- Canonical truth files: + - /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md + - /etc/pve/31_HOMELAB_REFERENCE.md +- Before any infra command, strictly check both files and relevant context blocks. +- Every infra command must print: + - ERROR_REGISTER_CHECK=OK + - REFERENCE_CHECK=OK + +### Production forum VM +- VMID: 160. +- Name: forum-prod. +- Node: pve02. +- IP: [PRIVATE_IP]. +- OS: Debian 12. +- Runtime: nginx, MariaDB 10.11, PHP 8.3 FPM, qemu-agent, cron, msmtp/sendmail transport. +- Final accepted snapshot: forum-final-accepted-091934Z. +- Mail final snapshot: forum-mail-ok-091815Z. +- Postlaunch snapshot: forum-postlaunch-ok-085501Z. + +### Public forums +- codevipe.ru -> Форум CodeVipe. +- gamevipe.ru -> Форум GameVipe. +- hkmods.ru -> Моды для Hollow Knight. +- zakrutim.ru -> Консервирование и закрутки. +- dsmods.ru -> Секреты и моды Doom. +- All five are public HTTPS 200 with valid TLS. +- Root/www DNS points through edge public IP 95.84.154.183. +- NPMplus on edge terminates TLS and proxies to http://[PRIVATE_IP]:80. + +### Mail +- Forum sender: noreply@pvepro.ru. +- Transport: XenForo sendmail -> msmtp -> mail.pvepro.ru. +- Built-in XenForo mail test reached Yandex. +- Yandex headers showed SPF pass and DKIM pass for pvepro.ru. +- Yandex placed the test in Spam with X-Yandex-Spam: 4. +- Classification: transport/authentication OK; remaining issue is sender reputation/content filtering. +- DNS change is not required from the captured header proof. +- Manual next action: click “Не спам!” in Yandex and add/check pvepro.ru in Yandex Postmaster. + +### Backups and restore +- Local backup configured inside VM160. +- Local archive path pattern: /var/backups/forums/forum-backup-*.tar.gz. +- pve02 Mail.ru Cloud fresh5/all-forums backup configured. +- New timer: forum-prod-fresh5-cloud-backup.timer active/enabled. +- Old codevipe-only timer disabled/inactive. +- Cloud remotes: + - pve02-mail-01-crypt + - pve02-mail-02-crypt + - pve02-mail-03-crypt + - pve02-mail-04-crypt +- Latest verified cloud stage during final acceptance: 20260701T083354Z. +- Restore drill passed: split parts verified, archive SHA matched, tar contained DB/file/config archives for all five forums. + +### XenForo jobs and cron +- cron daemon was missing, then installed and enabled. +- /etc/cron.d execution was proven by temporary cron proof. +- XenForo job runner configured: + - /root/scripts/forum-xenforo-run-jobs.sh + - /etc/cron.d/forum-xenforo-run-jobs +- Purpose: process XenForo jobs and cron tasks independent of visitor traffic. + +### DBTech/Tor timeout +- Each forum had one historical xf_error_log row for cURL timeout to check.torproject.org. +- Source addon: DBTech/Security DragonByte Security 5.0.0. +- dbtech_security_tornodes=0 on all five forums. +- Classified as non-blocking historical external timeout noise. +- Rows were not deleted. + +### Key final proofs +- /root/evidence/629_FORUM_PROJECT_FINAL_ACCEPTANCE_PROOF.txt +- /root/evidence/628_FORUM_MAIL_DELIVERABILITY_FINAL_SNAPSHOT_PROOF.txt +- /root/evidence/627_YANDEX_HEADER_DELIVERABILITY_CLASSIFICATION_PROOF.txt +- /root/evidence/622_TOR_TIMEOUT_CLASSIFICATION_FINAL_PROOF.txt +- /root/evidence/621_FORUM_CRON_DAEMON_JOBRUNNER_VERIFY_PROOF.txt +- /root/evidence/614_FORUM_POSTLAUNCH_BACKUP_MAIL_FINAL_ROLLUP_PROOF.txt +- /root/evidence/613_FORUM_FRESH5_CLOUD_RESTORE_DRILL_FINALIZE_PROOF.txt +- /root/evidence/612_FORUM_FRESH5_CLOUD_RESTORE_DRILL_CORRECTED_PROOF.txt +- /root/evidence/609_FORUM_PROD_FRESH5_CLOUD_BACKUP_FINALIZE_PROOF.txt +- /root/evidence/608_FORUM_PROD_FRESH5_CLOUD_BACKUP_FIRST_RUN_PROOF.txt +- /root/evidence/600_FORUM_PERSISTENT_SMTP_FINAL_STABLE_PROOF.txt +- /root/evidence/570_MAIL_DNS_FINAL_CLEAN_AUDIT_PROOF.txt +- /root/evidence/560_FORUM_BACKUP_SCRIPT_SETUP_PROOF.txt + +## PARKED_DOMAINS_PLACEHOLDER_PUBLIC_OK_20260701 + +- Parked/placeholder public page is live for three unused domains: newfi.ru, hapusya.ru and kingofwolk.ru. +- Public DNS root and www hostnames already point to edge public IP 95.84.154.183. +- Local placeholder service on edge-vm: homelab-parked-domains-http.service, enabled/active, serves marker PARKED_PAGE_OK on http://127.0.0.1:18088. +- NPMplus manual managed route files: /data/nginx/proxy_host/998.conf for newfi.ru, 997.conf for hapusya.ru and 996.conf for kingofwolk.ru. +- Public HTTPS validation from pve01 after final route-only finalize returned HTTP 200 and PARKED_PAGE_OK for all six hostnames: root and www for all three domains. +- Certificates were issued by certbot DNS-01 using Cloudflare API hooks; HTTP-01 must not be retried for these parked domains without a new plan because earlier attempts hit NPMplus default redirect/include-order behavior. +- Cloudflare token value is not stored in the reference; token file path only: /opt/npmplus/secure/parked_cf_token, root-owned mode 600. +- Certificate names on edge-vm: parked-newfi.ru, parked-hapusya.ru and parked-kingofwolk.ru; observed expiry during setup: 2026-09-29. +- Certbot deploy hook installed: /etc/letsencrypt/renewal-hooks/deploy/parked-domains-npmplus-deploy.sh. +- Health file: /var/lib/homelab-health/parked-domains-public.txt. +- Final route backup: /opt/npmplus/manual-backups/parked-stage16-route-only-20260701T155843Z. +- Final proof: /root/evidence/645_PARKED_DOMAINS_STAGE16_ROUTE_ONLY_FINALIZE_PROOF.txt. + +## GRAM1_ROOT_WWW_PLACEHOLDER_PUBLIC_OK_20260701 + +- gram1.ru root and www.gram1.ru are routed to the existing parked placeholder page on edge-vm. +- Existing gram1.ru subdomain routes were not changed. +- NPMplus managed route file: /data/nginx/proxy_host/995.conf. +- Upstream placeholder: http://127.0.0.1:18088. +- Certificate name on edge-vm: parked-gram1.ru. +- Certificate issuance mode: certbot DNS-01 with dedicated gram1 Cloudflare token file /opt/npmplus/secure/gram1_cf_token. +- Token value is not stored in reference; token file is root-owned mode 600. +- Certbot deploy hook: /etc/letsencrypt/renewal-hooks/deploy/gram1-root-npmplus-deploy.sh. +- Health file: /var/lib/homelab-health/gram1-root-placeholder.txt. +- Closure proof: /root/evidence/653_GRAM1_ROOT_WWW_PLACEHOLDER_CLOSE_STAGE19_PROOF.txt. + +## PVEPRO_ROOT_WWW_EDGE_LANDING_PUBLIC_OK_20260701 + +- pvepro.ru and www.pvepro.ru root/www were moved from the Mailcow VPS default web surface to an edge landing page. +- mail.pvepro.ru and nb.pvepro.ru were intentionally not changed. +- Cloudflare A records for pvepro.ru and www.pvepro.ru point to edge public IP 95.84.154.183. +- NPMplus managed route file: /data/nginx/proxy_host/994.conf. +- Landing service on edge-vm: homelab-pvepro-landing-http.service on http://127.0.0.1:18089. +- Landing marker: PVEPRO_LANDING_OK. +- Certificate name on edge-vm: landing-pvepro.ru. +- Certificate issuance mode: certbot DNS-01 with dedicated pvepro Cloudflare token file /opt/npmplus/secure/pvepro_cf_token. +- Token value is not stored in reference; token file is root-owned mode 600. +- Certbot deploy hook: /etc/letsencrypt/renewal-hooks/deploy/pvepro-root-npmplus-deploy.sh. +- Health file: /var/lib/homelab-health/pvepro-root-landing.txt. +- Closure proof: /root/evidence/658_PVEPRO_STAGE23_FINAL_READONLY_CLOSE_PROOF.txt. + +## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701 + +- taftauto.ru is the dacha router domain. +- Current public A record observed during audit: 194.33.48.131. +- Router model family: Netcraze-like, same as Moscow router family by operator statement. +- Safe private management access is not available yet. +- Do not expose the router admin interface publicly for certificate automation. +- Certificate auto-renewal/deploy to the router is intentionally blocked until a private access path exists. +- Recommended future path: establish VPN/NetBird/WireGuard or reverse-proxy/private management endpoint first; then issue DNS-01 certificate on a trusted node and deploy cert/key to the router only over that private path. +- Closure status: documented blocker, not runtime outage. +- Read-only audit proof: /root/evidence/654_PVEPRO_TAFTAUTO_EXTERNAL_READONLY_AUDIT_PROOF.txt. + +## DOMAIN_PORTFOLIO_FINAL_STATUS_20260701 + +- Public parked placeholder domains closed: newfi.ru, hapusya.ru, kingofwolk.ru. +- gram1.ru root and www.gram1.ru closed on the same placeholder page; existing gram1.ru service subdomains were not changed. +- pvepro.ru root and www.pvepro.ru closed on a separate edge landing page; mail.pvepro.ru and nb.pvepro.ru remain on the external VPS/IPs and were validated after the change. +- Forum domains remain XenForo on forum-prod through edge NPMplus: codevipe.ru, gamevipe.ru, hkmods.ru, zakrutim.ru, dsmods.ru. +- taftauto.ru is documented as blocked for certificate autodeploy until private router management access exists. +- Forum renewal one-shot proof from Stage18 returned OK. +- Parked certbot dry-run passed for newfi.ru and hapusya.ru; kingofwolk.ru dry-run hit transient Let’s Encrypt service-busy/rateLimited after a valid active certificate and working public HTTPS were already confirmed. +- Current finalization proof: /root/evidence/661_DOMAIN_PORTFOLIO_FINAL_CLOSE_AND_TAFTAUTO_BLOCKED_PROOF.txt. + +## TAFTAUTO_WG_PRIVATE_MANAGEMENT_OK_20260702 +- Public SSH 194.33.48.131:22 closed. +- WireGuard management path OK: edge [PRIVATE_IP] -> dacha router [PRIVATE_IP]. +- Router HTTP over WG returns HTTP/1.1 200 OK / Ndm-Sysmode: router. +- Proof: /root/evidence/665_TAFTAUTO_WG_PRIVATE_MANAGEMENT_OK_20260702_PROOF.txt + +## TAFTAUTO_WG_PSK_ROTATED_AND_MANAGEMENT_OK_20260702 +- WireGuard PSK rotated after leaked self-test. +- Dacha active interface after reimport: Wireguard1. +- Public SSH remains closed. +- Private management OK: edge [PRIVATE_IP] -> dacha [PRIVATE_IP] via WG. +- Proof: /root/evidence/666_TAFTAUTO_WG_PSK_ROTATED_AND_MANAGEMENT_OK_20260702_PROOF.txt + +## TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702 +- Supersedes proof 666 because proof 666 showed PUBLIC_SSH_22_STILL_OPEN. +- WireGuard PSK rotated. +- Public SSH closed. +- Private WG management OK: edge [PRIVATE_IP] -> dacha [PRIVATE_IP]. +- Proof: /root/evidence/667_TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702_PROOF.txt + +## TAFTAUTO_CERT_DNS01_ISSUED_20260702 +- DNS-01 certificate issued on edge-vm for taftauto.ru and www.taftauto.ru. +- Cert name: router-taftauto.ru. +- Cert path: /etc/letsencrypt/live/router-taftauto.ru/fullchain.pem. +- Key path: /etc/letsencrypt/live/router-taftauto.ru/privkey.pem. +- Proof: /root/evidence/668_TAFTAUTO_CERT_DNS01_ISSUED_20260702_PROOF.txt + +## TAFTAUTO_PUBLIC_EDGE_WG_ROUTER_FINAL_OK_20260702 +- Supersedes failed/partial proof 669 because direct SSH was open during that run. +- taftauto.ru and www.taftauto.ru point to edge public IP 95.84.154.183. +- NPMplus route: /opt/npmplus/nginx/proxy_host/993.conf. +- TLS terminates on edge using certbot DNS-01 cert router-taftauto.ru. +- Upstream is private WireGuard path: edge [PRIVATE_IP] -> dacha router [PRIVATE_IP]:80. +- Public HTTPS returns router page with x-taftauto-router: managed and ndm-sysmode: router. +- Direct dacha public SSH is closed; WG SSH remains open. +- Proof: /root/evidence/670_TAFTAUTO_PUBLIC_EDGE_WG_ROUTER_FINAL_OK_20260702_PROOF.txt + +## HOMEPAGE_EXTERNAL_SMARTAPE_AND_SITES_OK_20260702 +- Homepage services.yaml repaired with YAML-aware edit after bad indentation issue. +- SmartApe card added to External Homelabs. +- Router Moscow card removed. +- Sites category added with 11 site cards: CodeVipe, GameVipe, HKMods, Zakrutim, DSMods, Newfi, Hapusya, KingOfWolk, Gram1, PVEPro, TaftAuto. +- Proof: /root/evidence/671_HOMEPAGE_EXTERNAL_SMARTAPE_AND_SITES_OK_20260702_PROOF.txt + +## TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK_20260702 +- Certbot renew dry-run for router-taftauto.ru succeeds with manual DNS-01 hooks. +- Deploy hook installed: /etc/letsencrypt/renewal-hooks/deploy/taftauto-router-npmplus-deploy.sh. +- Deploy hook copies renewed cert/key into NPMplus /data/tls/router-certs/taftauto.ru and reloads nginx after nginx -t. +- Proof: /root/evidence/672_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK_20260702_PROOF.txt + +## TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_INSTALLED_DRYRUN_RATELIMITED_20260702 +- Supersedes partial proof 672. +- Deploy hook installed and manual deploy invocation works. +- NPMplus cert copy and nginx config validate OK. +- Public taftauto.ru remains OK. +- Certbot dry-run retry is deferred due Let's Encrypt rateLimited / Service busy. +- Proof: /root/evidence/673_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_INSTALLED_DRYRUN_RATELIMITED_20260702_PROOF.txt + +## VPN_GRAM1_ALIAS_ROUTES_REMOVED_20260702 +- Removed obsolete NPMplus route aliases without Homepage cards: auth.vpn.gram1.ru, gitea.vpn.gram1.ru, homepage.vpn.gram1.ru, kuma.vpn.gram1.ru. +- NPMplus nginx config validates after removal. +- Proof: /root/evidence/674_VPN_GRAM1_ALIAS_ROUTES_REMOVED_20260702_PROOF.txt + +## HOMELAB_HOMEPAGE_AND_TAFTAUTO_FINAL_OK_20260702 +- Homepage final layout/category state confirmed: YAML validates, categories use row/columns layout, logs show no YAMLException. +- TaftAuto public HTTPS works for taftauto.ru and www.taftauto.ru through edge/NPMplus. +- Direct public SSH to dacha router is closed. +- Private WG SSH to dacha router remains open. +- NPMplus nginx config validates. +- Proof: /root/evidence/677_HOMELAB_HOMEPAGE_AND_TAFTAUTO_FINAL_OK_20260702_PROOF.txt + +## CLOUDFLARE_TOKEN_AUDIT_20260702 +- Cloudflare token files audited without printing secrets. +- Token files exist, expected permissions were checked, tokens verify active, and expected zone access was checked. +- Proof: /root/evidence/678_CLOUDFLARE_TOKEN_AUDIT_20260702_PROOF.txt + +## EVIDENCE_REVIEW_INDEX_20260702 +- Read-only evidence index created for review/superseded proof cleanup planning. +- No evidence files were deleted or modified. +- Proof: /root/evidence/679_EVIDENCE_REVIEW_INDEX_20260702_PROOF.txt + +## EVIDENCE_SUPERSEDED_MAP_20260702 +- Evidence superseded map created. No evidence files were deleted. +- 666, 669, 672_TAFTAUTO_CERT_RENEW_DEPLOY_HOOK_OK, and 676 are not authoritative for final state. +- 677 is current final Homepage/TaftAuto authority; 678 is current Cloudflare token audit authority. +- Proof: /root/evidence/680_EVIDENCE_SUPERSEDED_MAP_20260702_PROOF.txt + +## HOMEPAGE_LOCAL_HEALTH_MONITORS_OK_20260702 +- Local edge health endpoint is used for Homepage monitoring of Роутер Москва and NPMplus. +- Роутер Москва href remains external, while siteMonitor uses http://[PRIVATE_IP]:18091/router-moscow. +- NPMplus siteMonitor uses http://[PRIVATE_IP]:18091/npmplus. +- Homepage YAML validates and logs show no YAMLException. +- Proof: /root/evidence/681_HOMEPAGE_LOCAL_HEALTH_MONITORS_OK_20260702_PROOF.txt + +## HOMEPAGE_SELF_CARD_REMOVED_FROM_CORE_20260702 +- Removed Homepage self-referential card from Core. +- Homepage YAML validates and logs show no YAMLException. +- Proof: /root/evidence/682_HOMEPAGE_SELF_CARD_REMOVED_FROM_CORE_20260702_PROOF.txt + +## CERT_RENEW_HOOKS_READONLY_AUDIT_20260702 +- Read-only audit of cert renewal cron entries, renewal configs, deploy hooks, and related script presence completed. +- No certbot renew/dry-run was executed. +- Proof: /root/evidence/683_CERT_RENEW_HOOKS_READONLY_AUDIT_20260702_PROOF.txt + +## CERT_RENEW_MAPPING_READONLY_AUDIT_20260702 +- Read-only mapping audit completed for renewal confs, deploy hooks, and forum renewal script. +- No certbot renew/dry-run was executed and no secrets were printed. +- Proof: /root/evidence/684_CERT_RENEW_MAPPING_READONLY_AUDIT_20260702_PROOF.txt + +## CERT_DEPLOY_HOOK_LINEAGE_GUARDS_OK_20260702 +- Deploy hooks hardened with RENEWED_LINEAGE guards for gram1, parked domains, and pvepro. +- TaftAuto hook already had lineage guard and remains guarded. +- bash -n validates all checked deploy hooks. +- No certbot renew/dry-run was executed. +- Proof: /root/evidence/685_CERT_DEPLOY_HOOK_LINEAGE_GUARDS_OK_20260702_PROOF.txt + +## PUBLIC_HTTPS_AND_CERT_EXPIRY_READONLY_20260702 +- Read-only public HTTPS and certificate expiry snapshot completed for forum, parked, gram1, pvepro, and taftauto domains. +- No certbot renew/dry-run was executed. +- Proof: /root/evidence/686_PUBLIC_HTTPS_AND_CERT_EXPIRY_READONLY_20260702_PROOF.txt + +## NPMPLUS_ROUTE_CERT_MAPPING_READONLY_20260702 +- Read-only NPMplus route to certificate mapping audit completed. +- Expected proxy routes were found and expected certificate files are present and valid for more than 30 days. +- No certbot renew/dry-run was executed. +- Proof: /root/evidence/687_NPMPLUS_ROUTE_CERT_MAPPING_READONLY_20260702_PROOF.txt + +## TODAY_PROOFS_SECRET_SCAN_READONLY_20260702 +- Read-only filename-only secret pattern scan completed for today proof files and active reference/error register. +- No matching secret value patterns were found. +- Proof: /root/evidence/688_TODAY_PROOFS_SECRET_SCAN_READONLY_20260702_PROOF.txt + +## HOMELAB_20260702_SESSION_CLOSURE_OK +- Session closure proof created for final OK chain 677-688. +- TaftAuto, Homepage, Cloudflare tokens, evidence map, cert hooks, deploy guards, public HTTPS, route/cert mapping, and secret scan are documented. +- Proof: /root/evidence/689_HOMELAB_20260702_SESSION_CLOSURE_OK_PROOF.txt + +## CROWDSEC_CAPI_NETBIRD_EXIT_ROUTE_20260702 +- edge-vm CrowdSec CAPI is registered and enabled through NetBird exit route. +- Client peer: edge-vm.netbird.selfhosted / 100.100.60.182. +- Primary exit routing peer: relay.netbird.selfhosted / 100.100.19.1 / Moldova. +- Backup egress peer present: mail.netbird.selfhosted / 100.100.147.204 / USA. +- NetBird exit route proof on edge-vm: wg allowed-ips includes 0.0.0.0/0 via relay; ip route get 1.1.1.1 uses wt0 table 7120. +- External trace after route: 85.121.4.192 / OTP, not home 95.84.154.183 / ARN. +- CrowdSec proof: cscli lapi status OK, cscli capi status OK, CAPI sharing/blocklist pull enabled, no DISABLE_ONLINE_API or -no-capi flags in active compose. +- WARP is intentionally absent and must not be reintroduced for this path. +- Unauthenticated https://api.crowdsec.net/v3/watchers/login returning HTTP 403 is expected network reachability proof. +- Final proof: /root/evidence/771AD_FINAL_CROWDSEC_CAPI_NETBIRD_CLOSURE_20260702T191057Z_PROOF.txt + +## HOMEPAGE_RU_LAYOUT_MONITORING_PROPOSALS_20260702 +- Homepage live config is on core-apps VM [PRIVATE_IP] under /opt/stacks/homepage/config. +- Groups/descriptions were normalized to Russian. +- Layout is row/6-columns for all active groups. +- siteMonitor fields are preserved; monitors must be repaired, not removed. +- Cloudflare card must be left untouched by explicit operator request. +- Red badges after migration were primarily caused by Homepage container resolving *.vpn.gram1.ru to edge NetBird IP and timing out; fixed by mapping vpn hostnames to edge LAN [PRIVATE_IP] in Homepage compose extra_hosts. +- Router Moscow monitor uses edge health endpoint http://[PRIVATE_IP]:18091/router-moscow. +- NPMplus monitor uses edge health endpoint http://[PRIVATE_IP]:18091/npmplus. +- Missing-card proposal report: /var/lib/homelab-health/homepage-missing-card-proposals-final.txt. + +## UPTIME_KUMA_MISSING_MONITOR_AND_PROPOSALS_20260702 +- Uptime Kuma lives on monitoring VM [PRIVATE_IP]. +- Added one missing monitor: NPMplus Edge Health -> http://[PRIVATE_IP]:18091/npmplus. +- Uptime Kuma DB backup was created before DB mutation under /root/uptime-kuma-manual-backups/773a-* on monitoring VM. +- DB integrity after insertion was OK. +- Deep monitoring proposal report: /var/lib/homelab-health/uptime-kuma-monitoring-proposals.txt. +- Cloudflare must not be touched by operator request. +- Recommended next monitors are P0 only after operator approval: Router Moscow Health, Backup/Restore Dashboard, Restore Drill Index, NetBird Peers, CrowdSec CAPI, Paperless Restore. + +## UPTIME_KUMA_NPMPLUS_ADMIN_REPAIR_20260702 +- Existing Uptime Kuma monitor "NPMplus Admin" was red because it checked https://[PRIVATE_IP]:81/. +- NPMplus admin is intentionally localhost-bound on edge-vm, so monitoring VM should not check the admin UI directly. +- Fixed monitor target to edge health endpoint: http://[PRIVATE_IP]:18091/npmplus. +- Monitor was repaired, not deleted. +- Cloudflare was not touched. + +## UPTIME_KUMA_APPROVED_MONITOR_BATCH_20260702 +- Approved Uptime Kuma monitor batch installed after Homepage/Kuma analysis. +- Existing NPMplus Admin monitor was repaired to http://[PRIVATE_IP]:18091/npmplus, not deleted. +- Edge health wrapper /usr/local/sbin/router-moscow-health provides safe endpoints on [PRIVATE_IP]:18091 for router-moscow, npmplus, loki, alloy, cadvisor, registry-cache, socket-proxy, diun, kopia and crowdsec. +- Cloudflare was not touched by explicit operator request. +- Uptime Kuma DB backups are under /root/uptime-kuma-manual-backups/773d-approved-batch-* on monitoring VM before the DB mutation. + +## UPTIME_KUMA_PENDING_HEALTH_MONITORS_FIX_20260702 +- Four pending Uptime Kuma keyword monitors were repaired by using local pve01 health endpoint paths instead of public backup.gram1.ru paths: + - Backup Restore Dashboard -> http://[PRIVATE_IP]:9101/backup-restore-dashboard.txt + - Restore Drill Index -> http://[PRIVATE_IP]:9101/restore-drill-index.txt + - Paperless Restore -> http://[PRIVATE_IP]:9101/paperless-restore.txt + - AdGuard Rewrite Sync -> http://[PRIVATE_IP]:9101/adguard-rewrite-sync.txt +- Cloudflare was not touched. +- Monitors were repaired, not deleted. + +## UPTIME_KUMA_FOUR_HEALTH_SOURCES_REPAIRED_20260702 +- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR. +- Repaired pve01 health source files under /var/lib/homelab-health: + - backup-restore-dashboard.txt + - restore-drill-index.txt + - paperless-restore.txt + - adguard-rewrite-sync.txt +- Kuma monitors point to local pve01 health endpoints on http://[PRIVATE_IP]:9101/. +- Cloudflare was not touched. +- Monitors were repaired, not deleted. + +## DOCKGE_DISCONNECTED_HOSTS_AND_STALE_STACKS_20260702 +- Dockge is currently present only on edge-vm. +- core-apps and monitoring run Docker containers but have no Dockge/Dockge-agent detected. +- Dockge "2 not connected" should be interpreted as two disconnected Docker hosts unless later evidence shows otherwise. +- No stopped containers were found across edge-vm, core-apps and monitoring during analysis. +- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring. +- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification. + +## DOCKGE_REMOTE_AGENTS_INSTALLED_20260702 +- Dockge main instance runs on edge-vm [PRIVATE_IP]. +- Remote Dockge agents were installed on: + - core-apps [PRIVATE_IP]:5001 + - monitoring [PRIVATE_IP]:5001 +- Edge Dockge agent table stores the two agent URLs with username/password; password must never be printed. +- No runtime containers were deleted. +- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first. + +## DOCKGE_REMOTE_AGENTS_AND_STALE_ARCHIVE_FINAL_20260702 +- Dockge main instance: edge-vm [PRIVATE_IP]. +- Remote agents connected: core-apps [PRIVATE_IP]:5001, monitoring [PRIVATE_IP]:5001. +- Stale moved edge stack folders archived under /opt/dockge-stale-archive/20260702T230442Z on edge-vm; no runtime containers were deleted. +- External remote stacks exposed to Dockge using bind mounts: /opt/vaultwarden-compose, /opt/gotify-compose, /opt/uptime-kuma-compose into /opt/stacks. +- Final target: remote not_visible count must be 0 and edge stale remaining matches must be 0. + +## DOCKGE_FINAL_MANAGED_STACKS_20260702 +- Dockge main stack, npmplus and remote dockge-agent stacks were recreated from /opt/stacks so Dockge can manage them. +- Edge Dockge listens on 127.0.0.1:5001; LAN probe to [PRIVATE_IP]:5001 may return 000 and is not the correct health proof. +- Remote agents listen on [PRIVATE_IP]:5001 and [PRIVATE_IP]:5001. +- Stale moved stacks were archived, not deleted. + +## SERVICE_SETUP_ROADMAP_DEEP_20260702 +- Configuration order: secrets, DNS/proxy/TLS, backup/restore, monitoring, SSO, docs, core data apps, sensitive apps, productivity, media, automation, utilities. +- Do not configure data-heavy or automation services before backup and monitoring are proven. +- Per-service done criteria: URL, TLS, admin, SSO decision, volumes, backup, restore test, Kuma, Homepage, docs. + +## PHASE00_FREEZE_CLOSED_20260702 +- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain. +- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml. +- Old systemd unit homelab-registry-cache.service was disabled and masked because it recreated the container without Compose labels. +- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/. +- Next phase: Vaultwarden and break-glass secret baseline. + +## PHASE00_FREEZE_CLOSED_20260702 +- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain. +- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml. +- Old systemd unit homelab-registry-cache.service was archived and masked because it recreated the container without Compose labels. +- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/. +- Next phase: Vaultwarden and break-glass secret baseline. + +## PHASE00_FREEZE_CLOSED_20260702 +- Service inventory normalized: no manual containers and no compose projects outside /opt/stacks remain. +- Registry cache is managed by Dockge/Compose: project registry-cache, compose /opt/stacks/registry-cache/compose.yaml. +- Old systemd unit homelab-registry-cache.service was archived and masked with /dev/null because it recreated the container without Compose labels. +- Correct registry health URL is http://[PRIVATE_IP]:5000/v2/. +- Next phase: Vaultwarden and break-glass secret baseline. + +## OPERATOR_RULE_CLOSE_TAILS_IMMEDIATELY_20260714 +- Жёсткое правило: хвосты не оставлять. Любая проблема, временный артефакт, неудачная проверка, блокер или анти-регрессия, обнаруженные в текущем scope, должны быть устранены и закрыты до перехода к следующей задаче. +- После любой неуспешной команды текущая задача остаётся активной до установления точной причины, исправления, проверки исправления, добавления анти-регрессии, очистки временных артефактов и выпуска закрывающего proof. +- Этап разрешено помечать CLOSED только при UNRESOLVED_TAIL_COUNT=0, BLOCKER_COUNT=0, TEMPORARY_ARTIFACT_COUNT=0, проверенном rollback, зелёном health, обновлённых proof и reference. +- Запрещено откладывать выявленный хвост только ради удобства или перехода к следующему этапу. +- Если технически необходим отдельный подэтап, он должен быть ограничен точным scope, выполнен и закрыт немедленно до возврата к основной работе. +- Исключения: внешняя зависимость, опасная неоднозначность, риск раскрытия секрета, отсутствующий обязательный файл или явное решение пользователя. В таком случае статус должен быть BLOCKED или OPEN с точным блокером; статус CLOSED запрещён. +- Следующий этап не начинается, пока текущий хвост не закрыт. +- Proof: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/report.txt +- Secret scan: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/strict-secret-scan.txt + +## CLUSTER_ADMIN_STAGE4H_SCOPE_DEFINED_20260714 +- Stage4G остаётся закрытым и стабильно работает в режиме observe-notify. +- Закрыты два хвоста dependency-аудита: глобальный LIMIT всего UNION ALL и узкий шаблон collector_patch_required. +- Migration plan содержит collector_patch_required=true; dynamic contract содержит collector_patch_required_after_stage4c=true. Это два подтверждения одного обязательного требования. +- Migration 003 не применена, 12 новых столбцов отсутствуют. +- Migration выполняет полный UPDATE observations, четыре SET NOT NULL, пять VALIDATE CONSTRAINT и пять неконкурентных CREATE INDEX. +- Текущий collector не формирует обязательный provenance-набор; готовый collector patch отсутствует. +- Production adapters и production apply phase отсутствуют. +- Отдельный apply migration 003 без collector patch запрещён. +- Безопасный порядок: collector patch candidate → временная БД и acceptance/rollback → контролируемый migration+collector cutover → adapter integration. +- Текущий этап открыт: 4H_COLLECTOR_PROVENANCE_PATCH_AND_MIGRATION003_PREAPPLY_READINESS. +- Предыдущие хвосты закрыты; Stage4H не считается CLOSED до реализации и полного seal. +- Proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt +- Secret scan: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/strict-secret-scan.txt + +## CLUSTER_ADMIN_STAGE4H_COLLECTOR_BASELINE_AUDIT_CLOSED_20260714 +- Исправленный baseline-аудит collector завершён. +- collector.py: 113 строк, SHA256 подтверждён. +- Привилегированные файлы нельзя читать через caller-side `< file` перед sudo; применён sudo wc без внешнего перенаправления. +- Фактические поля collector_runs: finished_at и error_text; completed_at и error отсутствуют. +- Найдены один canonical source instance и один collector_runs_foundation_enrich_trg. +- Trigger обогащает collector_runs полями source_instance_id и run_key. +- Текущий collector ещё не пишет provenance-поля observations; migration 003 не применена. +- Production осталась 0|0|OK, timer активен, failed units отсутствуют. +- Baseline-аудит закрыт без хвостов; Stage4H остаётся OPEN для isolated collector provenance patch candidate. +- Proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt +- Secret scan: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/strict-secret-scan.txt + +## CLUSTER_ADMIN_STAGE4H_COLLECTOR_PATCH_CANDIDATE_CLOSED_20260714 +- Из точного live collector создан isolated provenance patch candidate; production collector не изменён. +- Candidate пишет все 12 migration003 provenance-полей и применяет SHA256 fallback-дедупликацию. +- Metadata разрешает только instance, job, mountpoint и device, ограничена по типам и размеру. +- Candidate скомпилирован локально и проверен на VM180 Python 3.11.2 от clusteradmin. +- SSH stdin harness исправлен: bash -s получает ровно path, SHA256 и bytes; первый аргумент дополнительно проверяется по безопасному шаблону. +- Self-test и отрицательный CLI-тест RC64 прошли; временный remote-файл удалён. +- Production осталась 0|0|OK; live collector и timer не изменены. +- Неудачная попытка закрыта без хвостов; Stage4H остаётся OPEN до temporary-DB acceptance и controlled cutover. +- Candidate root: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z +- Proof: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/report.txt +- Secret scan: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/strict-secret-scan.txt + +## OPERATOR_RULE_NO_OVERSIZED_MONOLITH_COMMANDS_20260714 +- Запрещены чрезмерно длинные интерактивные однострочные команды с глубокой вложенностью SSH, SQL, Python, awk и кавычек. +- Рекомендуемый предел интерактивной команды: 8000 байт. Превышение допускается только для простого текста без вложенных языков. +- Сложная операция оформляется как отдельный versioned task-скрипт с contract, syntax-check, dry-run, manifest, rollback и proof. +- Создание task-скрипта и его запуск выполняются разными короткими командами. +- Remote stdout/stderr всегда сохраняются до проверки RC; запрещено терять вывод из-за errexit-sensitive command substitution. +- Перед запуском проверяются SHA256, размер, владелец, режим и синтаксис task-скрипта. +- Обрезанная, повреждённая или частично вставленная команда считается невыполненной; сначала проводится side-effect audit. +- Хвост parse-ошибки закрывается только после доказательства отсутствия изменений и временных артефактов. +- Proof: /var/lib/homelab-change-sets/operator-command-size-guard-v1/20260714T182653Z/report.txt + +## HOMELAB_ADMIN_CLI_CONTRACT_20260714 +- Поддерживаемые команды: version, status, lint TASK_DIR, validate TASK_DIR, run TASK_DIR PHASE. +- Отдельная команда --help отсутствует. Неизвестный аргумент печатает usage и штатно возвращает RC=64. +- Нельзя трактовать RC=64 от --help как неисправность runner. +- Для проверки доступности runner использовать homelab-admin version; для структуры CLI использовать зафиксированную usage-строку. +- Proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v1/20260714T183338Z/report.txt + +AUTOMATIC_CLUSTER_CONTEXT_REFRESH_20260721 +Collector: /usr/local/sbin/homelab-context-collect +Scheduler: hourly at minute 17 via /etc/cron.d/homelab-context-refresh +Destination: immutable Gitea runtime history with CONTEXT-AUTO UTC command IDs. + +AUTOMATIC_CLUSTER_CONTEXT_REFRESH_DIRECT_20260721 +Publisher: /usr/local/sbin/homelab-context-refresh-direct +Schedule: hourly at minute 17. +Destination: immutable assistant-runtime history with CONTEXT-AUTO UTC IDs. +DETAIL=SECTION_REFERENCE_REGISTER_END +DETAIL=SECTION_ERROR_REGISTER_BEGIN +# HOMELAB ASSISTANT ERROR REGISTER + +Назначение: перед каждой следующей командой сверяться с этим файлом. + +## Критические ошибки ассистента +1. Повторно дал слишком большой интерактивный paste в shell. +2. Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт. +3. Дал генератор справочника прямо в терминал вместо безопасного маленького шага. +4. Нарушил своё же правило: не давать длинные вложенные команды с кавычками. + +## Жёсткие правила перед каждой командой +CHECK-1: команда не должна быть большим paste. +CHECK-2: команда не должна содержать большой here-doc. +CHECK-3: команда не должна смешивать Markdown, backticks и shell-логику. +CHECK-4: команда не должна иметь вложенный ssh с несколькими уровнями кавычек. +CHECK-5: команда не должна печатать секреты. +CHECK-6: если создаётся файл, сначала маленький безопасный шаг, потом проверка. +CHECK-7: если команда длиннее 8 строк, её нельзя давать в интерактивный shell. +CHECK-8: для edge-vm использовать debian@[PRIVATE_IP] и sudo. +CHECK-9: для forum-prod использовать pve02 и ключ [SENSITIVE_PATH] +CHECK-10: Corosync не трогать без отдельного плана и rollback. + +## Текущие важные факты +Internal network: [PRIVATE_IP]/24. +Migration config: migration: secure,network=[PRIVATE_IP]/24. +Corosync remains on [PRIVATE_IP]/12/13. +VM160 forum-prod is not in Proxmox nightly backup. +VM130 edge-vm has secondary disk backup=0 risk. +05_edge_compose_safe.tgz quarantined. + +## Правило для справочника +Не генерировать большой справочник через интерактивную вставку. +Следующий справочник делать только маленькими append-блоками или через файл, созданный вне интерактивного shell. + +11. Ошибка: считать offhost OK после failed rsync. +Если rsync/scp упал, нельзя проверять latest offhost-файл без сверки имени. +Проверка должна подтвердить именно новый архив, например дату 2026_06_30-00_20_59. +Старый OFFHOST_ZSTD_OK не закрывает новый backup. + +12. Ошибка: широкий secret-поиск по /opt/stacks дал шум. +Не искать Cloudflare/SMTP сразу по /opt/stacks, MkDocs site, backups и HTML-отчётам. +Для ротации сначала смотреть live-конфиги: /opt/npmplus/tls/certbot/renewal/*.conf и реальные credentials-файлы из них. +Значения секретов не печатать; выводить только пути, ключи и redacted-поля. + +13. Ошибка: SQL с одинарными кавычками внутри одинарной SSH-команды ломается. +Нельзя писать sqlite SQL вида j.type in ('object','array') внутри ssh '...'. +Для sqlite через ssh использовать SQL без одинарных кавычек: char(36), length(j.atom), двойные внешние кавычки, либо отдельный файл. +Перед запуском проверять, что команда не содержит конфликтующих уровней кавычек. + +14. Ошибка: путать контекст входа и узел выполнения. +Стартовая точка оператора: root@pve01 / [PRIVATE_IP]. +edge-vm: ssh debian@[PRIVATE_IP], внутри использовать sudo. +pve02/pve03: ssh root@pve02 или ssh root@pve03. +forum-prod: заходить через pve02, ключ [SENSITIVE_PATH] +Перед каждой командой явно понимать: где выполняется локальная часть, где remote-часть, каким пользователем. + +15. Ошибка: повторно нарушено правило №13 после его добавления. +Снова был использован SQL JSON-path в одинарных кавычках внутри ssh '...'. +Запрещено использовать в remote sqlite SQL фрагменты вида '$.key' или '.key'. +Для JSON-path использовать только char(...), без одинарных кавычек внутри SQL. +Команду с ошибкой char(36)||.dns_provider считать битой и не использовать. + +16. Основополагающее правило: перед каждой командой ассистент обязан явно написать: Сверка с файлом ошибок: CHECK пройден. +После этой строки можно давать только одну короткую команду или один логический блок команд. +Нельзя выдавать команды без предварительной сверки с этим файлом ошибок. +Нельзя продолжать после собственной ошибки без записи ошибки в этот файл. + +19. Уточнение формата: не писать отдельную строку сверки перед каждой командой в пакете. +Сверка должна быть внутри самой команды первым логическим блоком: grep правил из файла ошибок и RULE_CHECK_OK. +Разрешено давать до трёх логически связанных команд сразу, по возможности в одну строку каждая. +Не писать фразу "ждём" после команд; указывать только контрольные строки результата. + +20. Ошибка: широкий XenForo SMTP-поиск по src дал шум исходников. +Для XenForo SMTP не grep-ать весь /src; сначала смотреть live src/config.php без значений, затем БД XenForo и xf_option/emailTransport. +Значения DB-паролей и SMTP-паролей не печатать. + +21. Ошибка: nested PHP php -r дал Parse error на forum-prod. +Команды 102-104 считать битыми: они не подтвердили и не изменили smtpSsl. +Не продолжать длинные вложенные ssh+php -r с кавычками; для forum-prod лучше войти в VM или использовать короткие mysql/php команды без вложенного кода. +Рабочими считать: DB backup перед SMTP, SMTP password update, JSON postcheck; SMTP auth ещё не закрыт из-за CODE=530 на AUTH LOGIN. + +22. Ошибка: самодельный base64 PHP для SMTP auth сломан. +Команда 108 дала PHP Parse error на smtpHost и пустой proof-файл. +Не использовать больше сгенерированные большие base64 PHP для SMTP/XenForo. +Для XenForo SMTP использовать короткие mysql-запросы к xf_option и готовые штатные механизмы XenForo. + +23. Ошибка: MariaDB не поддержала cast('true' as json) в JSON_SET. +Команда 111 не изменила smtpSsl: OLD_SMTPSSL=false, SQL ERROR 1064, NEW_SMTPSSL=false. +Перед боевым UPDATE сначала проверять JSON_SET синтаксис на тестовом JSON через SELECT. +Не считать PHP_FPM_RELOAD_OK подтверждением изменения БД. + +24. Ошибка: exit 1 в interactive-check закрыл SSH-сессию. +Нельзя использовать { echo RULE_CHECK_FAIL; exit 1; } в командах, выполняемых прямо в интерактивном root@pve01 shell. +При failed-check использовать безопасный шаблон: if grep ...; then echo RULE_CHECK_OK; ...; else echo RULE_CHECK_FAIL; fi. +Не делать fragile grep по точной строке с кавычками, если в файле ошибок строка содержит shell/SQL quoting. + +25. Ошибка анализа: не надо слепо включать smtpSsl=true при smtpPort=587. +smtpPort=587 обычно означает STARTTLS, а не implicit SMTPS. +CODE=530 на AUTH LOGIN означает, что ручной checker не сделал STARTTLS; это не доказывает неверный SMTP-пароль. +Сначала проверять AUTH через openssl s_client -starttls smtp, без вывода секрета. + +26. Ошибка: openssl -crlf вместе с ручным CRLF ломает SMTP AUTH checker. +Команда 116 получила 535 Invalid base64 data in continued response после 334 Username. +Это указывает на битую base64-строку checker-а, а не на неверный SMTP-пароль. +Не использовать -crlf, если команды уже отправляются с явным \r\n. +Для 587 оставлять smtpSsl=false и проверять STARTTLS корректным checker-ом. + +27. Ошибка: Python heredoc внутри ssh сломал not_ok диагностику. +Команда 138 пересобрала dashboard, но диагностический Python-блок упал с NameError из-за кавычек. +Не использовать heredoc Python внутри вложенного ssh; для dashboard verification использовать cat/grep JSON или простые команды. + +28. Ошибка: docker compose ls --format json | wc -l дал ложный COMPOSE_PROJECT_COUNT=1. +Причина: JSON-вывод может быть одной строкой/структурой, wc -l не является счётчиком проектов. +Для Docker Compose count использовать обычный docker compose ls --all + awk по строкам таблицы или JSON parser, не wc -l. + +29. Ошибка: monitoring compact status искал неверные имена health-файлов. +Факт: npmplus-cert-expiry.txt не является подробным файлом; подробный файл называется npmplus-certificate-expiry.txt, а статусный alias npmplus-cert-expiry-health.txt. +Факт: health-metrics.txt не обязан существовать как health-файл; health metrics могут публиковаться через node-exporter textfile collector. +Для Prometheus targets проверять HTTP_CODE/RC явно, не оставлять пустой блок PROMETHEUS_TARGETS_LOCAL. + +30. Ошибка/инцидент: Prometheus API HTTP_CODE=000 после monitoring reference. +Факт: в monitoring compact status curl к 127.0.0.1:9090 вернул HTTP_CODE=000, а raw inventory не показал контейнер prometheus среди monitoring containers. +Нельзя считать monitoring слой runtime-closed только по secret scan/reference proof; нужно отдельно проверить Prometheus container/API и обновить справочник по факту. + +31. Строгое правило: перед каждой командой сверяться с файлом ошибок и справочником. +Перед любым действием команда обязана проверять /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md и текущий 31_HOMELAB_REFERENCE.md. +Команда обязана печатать ERROR_REGISTER_CHECK=OK и REFERENCE_CHECK=OK до основного действия. +Если сверка не пройдена, команда не должна выполнять основное действие. Не использовать exit 1, чтобы не закрывать SSH-сессию. + +32. Ошибка: 153_PROMETHEUS_TARGETS_SETTLED_PROOF содержит SyntaxError из-за вложенного Python внутри SSH. +Факт: Python -c внутри ssh потерял кавычки вокруг /tmp/prom-targets-settled.json, data, activeTargets, labels, job, health. +Файл 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt нельзя считать валидным settled proof. +Для таких проверок не использовать вложенный Python; использовать curl + grep/sed/awk либо отдельный локальный файл-скрипт. + +34. Ошибка операционного ввода: полный терминальный лог MobaXterm попал в shell. +Факт: bash пытался выполнить строки Authenticating, MobaXterm banner, RULE_CHECK_OK, PROOF_OK и таблицы вывода как команды. +Такие выводы не являются инфраструктурным состоянием; считать невалидными только соответствующие failed command attempts. +Дальше давать короткие команды и не вставлять обратно полный transcript в shell. + +33. Security finding: root authorized_keys на PVE-нодах имел права 777. +Факт: inventory 158 показал [SENSITIVE_PATH] и backup-файлы с режимом 777 на pve01/pve02/pve03. +Нужно исправлять на [SENSITIVE_PATH]=700 и authorized_keys/authorized_keys.bak*=600, затем фиксировать before/after proof. + +35. Ошибка проверки: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED не доказал исправление прав. +Факт: AFTER всё ещё показал 777 для [SENSITIVE_PATH] на pve01/pve02/pve03. +Возможная причина: authorized_keys является symlink, а stat без -L показывает режим ссылки 777. +Нужно проверять stat -L целевого файла и считать 160 недостаточным proof. + +36. Quality check: Storage block needs integrity and pve03 capacity coverage review. +Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL. +Факт: pve03 /mnt/staging observed at 77%, close to WARN threshold 80%, while disk-space.txt did not visibly list pve03_staging. +Before closing storage layer, verify block integrity and pve03 capacity coverage. + +37. Coverage gap: pve03_staging missing from disk-space health coverage. +Факт: 169 showed pve03 /mnt/staging at 77%, close to WARN 80%, while disk-space.txt listed pve01_staging, pve02_staging and edge_immich_media only. +Before closing storage fully, add or document pve03_staging capacity monitoring. + +38. Quality check: Service Dependency Map block needs integrity review. +Факт: terminal output around command 222 showed paste artifact near "FAIL; fi". +Before closing Service Dependency Map, verify 178_REFERENCE_SERVICE_DEPENDENCY_MAP_BLOCK.txt and the reference section for shell/log garbage or truncation. + +39. Ошибка проверки: post deep audit BAD_MARKER scan caught prose false positive. +Факт: строка справочника с фразой No unresolved final REVIEW_REQUIRED... SyntaxError... была принята за BAD_MARKER. +Это не runtime/log мусор, а документированная фраза deep audit closure. В scan нужно исключать эту prose-строку. + +40. Ошибка/вывод: VM150 Mail cloud single-file upload failed with 413. +Факт: Mail/WebDAV rejected one 6.6G vzdump file with 413 Request Entity Too Large. +Решение: use VM150 recurring chunked script with 512M parts and download verification. + +41. Ошибка: post-backup-pass-slo-reconciliation checked pve01 router health on edge-vm. +Факт: first reconciliation wrote STATUS=ERROR because router-running-config-mail-cloud.txt is on pve01, not edge-vm. +Fix: build reconciliation from pve01 and edge-vm health files by correct owner. + +## ASSISTANT_COMMAND_BATCHING_RULE_20260630 +- Rule: combine related infrastructure checks, proof generation, reference updates and validation scans into one safe batch command when practical. +- Do not split work into many small commands when the operations are low-risk, read-only, or naturally part of one evidence block. +- Still keep mandatory preflight checks, short bounded commands, no large paste, no here-docs, no interactive shell traps, and no sensitive values in output. +- Split into smaller commands only when safety, rollback, quoting risk, or troubleshooting clarity requires it. + +## HOME_PORTAL_BASE64_APPLY_FAILURE_20260630 +- Prior home-portal apply attempt 312 failed because a large base64 Python payload decoded as invalid UTF-8. +- Rule: avoid large opaque base64 script payloads for homelab changes; use transparent bounded shell/perl/python commands instead. + +## HOMELAB_COMMAND_SAFETY_HARDENING_20260630 +- Rule: do not use large opaque payloads, generated base64 scripts, or hidden multi-line script blobs for homelab changes. +- Use transparent bounded shell/perl/python commands, or create a temporary readable script and validate it before execution. +- Any script used for changes must pass a syntax/parse check before it is allowed to modify files or restart services. +- Any Traceback, UnicodeDecodeError, SyntaxError, Permission denied, command-not-found, nonzero critical RC, or failed write marker must force REVIEW or FAILED, never OK. +- Success requires both runtime health and content-specific post-checks: expected files changed, expected cards exist, duplicates absent, service/container alive, and no new critical log pattern. +- HTTP 200 alone is not a success condition for configuration changes. +- If an apply step fails, the next command must explicitly record the failed attempt and then run a corrected transparent apply command. + +## HOME_PORTAL_API_ERROR_CLOSURE_RULE_20260630 +- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200. +- If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed. +- External informational widgets such as weather/Open-Meteo must not block the service launcher portal. + +## HOME_PORTAL_COMPLEX_QUOTING_FAILURE_20260630 +- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('. +- Cause: command was too complex and fragile due to nested shell/perl/python quoting. +- Rule: do not use deeply nested one-liners for config edits; first run short state checks, then apply small transparent edits. + +## HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RULE_20260630 +- Rule: Homepage cleanup validation must check active config files only, not logs or backup files. +- Active files are services.yaml, bookmarks.yaml, widgets.yaml, settings.yaml and docker.yaml under /opt/stacks/homepage/config. +- Matches in /logs/ or *.bak* are historical evidence only and must not make cleanup REVIEW unless the active config still contains the target. + +## HOME_PORTAL_LINK_VALIDATION_NPMPLUS_DEFAULT_RULE_20260630 +- Rule: Homepage link validation must not treat HTTP 200 as success if the body is the NPMPlus default “Congratulations” / host-not-set-up page. +- Green siteMonitor dots can be false positives when NPMPlus returns its default vhost page. +- A portal card is openable only when DNS, reverse-proxy host mapping, and target content are all valid. + +## ROUTER_CLI_STDIN_APPLY_FAILURE_20260630 +- Context: attempted Netcraze router ACL apply through SSH stdin/multiline for Homepage Moscow Router monitor fix. +- Evidence: 391_MOSCOW_ROUTER_EDGE_VM_WEBADMIN_ACL_APPLY_PROOF.txt. +- Failure: STDIN_SHOW_OK=no, APPLY_SKIPPED=stdin_show_failed, CONFIG_HAS_EDGE_5080=no, CONFIG_HAS_EDGE_5083=no. +- Rule: do not apply Netcraze config through stdin/multiline SSH; prove CLI input mode first. + +## ROUTER_CLI_PROBE_STDIN_CONSUMPTION_ERROR_20260630 +- Context: ACL syntax read-only probe loop executed only one command because ssh consumed the loop stdin. +- Evidence: 393_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_PROOF.txt showed COMMAND_COUNT=1. +- Fix pattern: use ssh -n or redirect SSH stdin away from the command-list loop for all future SSH-in-loop probes. + +## ROUTERBACKUP_READONLY_CLI_PERMISSION_LIMIT_20260630 +- Context: routerbackup user can run show running-config but cannot execute configuration commands such as access-list. +- Evidence: 394_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_FIXED_PROOF.txt. +- Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli]. +- Rule: do not attempt ACL/config changes as routerbackup; use admin/operator router access or router Web UI for write changes. + +## PROOF_SUMMARY_EXTRACTION_BLANK_20260630 +- Context: proof 395 tried to read COMMAND_COUNT/ZERO_RC_COUNT from proof files 393/394, but those summary fields were printed to console and were not persisted inside the proof files. +- Evidence: 395_ROUTER_CLI_ERROR_REGISTER_UPDATE_PROOF.txt has blank P393_COMMAND_COUNT/P394_COMMAND_COUNT/P394_ZERO_RC_COUNT fields. +- Rule: when creating follow-up proof summaries, derive counts from persisted proof body lines such as CMD_INDEX and CMD_RC, or write summary fields into the proof file itself. + +## ROUTER_ACL_UI_RULE_DELETE_AND_RESTORE_20260701 +- Context: while moving Moscow Router monitor rules to _WEBADMIN_Bridge1, two _WEBADMIN_Bridge0 Nextcloud Talk TURN rules were accidentally removed. +- Deleted/restored rules: Home [PRIVATE_IP]/24 to Nextcloud [PRIVATE_IP] TCP/UDP 3478. +- Verification proof: 399_ROUTER_ACL_RESTORE_AFTER_UI_PROOF.txt. +- Rule: after manual router Web UI ACL edits, compare _WEBADMIN_Bridge0, _WEBADMIN_Bridge1 and _WEBADMIN_GigabitEthernet1 before continuing. + +## ROUTER_MONITOR_NODE_LOOKUP_PROBE_BUG_20260701 +- Context: proof 400 tested Moscow router HTTPS monitor from Homepage container with custom Node lookup callback. +- Failure: NODE_HTTPS_RESOLVE_ERROR=ERR_INVALID_IP_ADDRESS and NODE_HTTPS_RESOLVE_INSECURE_ERROR=ERR_INVALID_IP_ADDRESS. +- Impact: Node HTTPS resolve subtest is invalid; curl --resolve results remain valid. +- Rule: do not rely on custom Node lookup callback until separately tested; prefer curl --resolve or simple direct Node HTTP checks. + +## ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701 +- Context: Moscow Router Homepage monitor after ACL fix. +- Evidence: proofs 399, 400, 401. +- Observed: edge-vm TCP to [PRIVATE_IP]:5080/5083 is OK, but router HTTP/HTTPS returns 403 for all tested paths. +- Impact: Homepage siteMonitor cannot be made green via direct router HTTP URL until router web service allows the Bridge1/Proxmox/protected segment or an explicit safe monitor endpoint is used. +- Rule: do not keep changing ACL for this symptom; ACL is already correct. Fix service access scope or use a dedicated monitor endpoint. + +## PY_COMPILE_PYC_PERMISSION_ERROR_20260701 +- Context: installing edge-vm Moscow router health endpoint. +- Mistake: assistant used `python3 -m py_compile` on a script installed under /usr/local/sbin as root. +- Actual impact: non-root syntax check attempted to create /usr/local/sbin/__pycache__ and failed with Permission denied. +- Recovery: validate syntax with `ast.parse` instead of py_compile, because it does not write .pyc files. +- Rule: do not use py_compile against root-owned system paths from an unprivileged user. + +## ROUTER_HOME_IP_DOCKER_ROUTE_CONFLICT_20260701 +- Context: Moscow Router Homepage siteMonitor attempt using http://[PRIVATE_IP]:5080. +- Evidence: proof 405 showed route to [PRIVATE_IP] via Docker bridge and Homepage container ECONNRESET. +- Impact: do not use [PRIVATE_IP] directly as Homepage siteMonitor from edge-vm. +- Mitigation: use dedicated edge-vm health endpoint that checks router TCP [PRIVATE_IP]:5080 and returns HTTP 200/503 for Homepage. + +## HOMEPAGE_ROUTER_MOSCOW_YAML_TITLE_SHAPE_MISMATCH_20260701 +- Context: applying Moscow Router Homepage siteMonitor health endpoint. +- Mistake: assistant apply script matched only property-style cards with `title`, but active Homepage YAML can use service-name-key style like `- Service Name:`. +- Evidence: proof 407 returned YAML_CHANGE_COUNT=0 and APPLY_SKIPPED=unexpected_match_count while health endpoint proof 406 was OK. +- Recovery: use YAML-aware updater that handles both property-style and service-name-key style cards. + +## HOMEPAGE_ROUTER_MOSCOW_APPLY_SCRIPT_SYNTAX_ERROR_20260701 +- Context: applying Moscow Router Homepage health endpoint. +- Mistake: assistant generated Python script with invalid f-string escaping. +- Evidence: proof 408 showed SyntaxError in /tmp/homepage-router-moscow-apply-fixed.py. +- Actual impact: YAML was not changed, so Homepage green dot could not appear. +- Recovery: stop applying blind updater; first discover exact active YAML shape, then apply minimal YAML-aware edit. +- Rule: validate generated Python syntax locally before scp/remote execution; avoid f-strings in generated maintenance scripts. + +## FORUM_PROD_VM160_FIRST_BOOT_SSH255_20260701 +- Context: clean rebuilt VM160 first boot. +- Symptom: PING_OK=yes and TCP22_OK=yes, but SSH_RC=255. +- Rule: do not proceed with OS baseline until SSH failure is diagnosed; likely old known_hosts key or cloud-init/root-key issue. + +## FORUM_PROD_VM160_NESTED_HOSTNAME_PROOF_QUOTING_20260701 +- Context: VM160 first SSH proof after rebuild. +- Issue: command substitution $(hostname) inside nested ssh was expanded on pve02 before entering VM160. +- Impact: VM_HOSTNAME=pve02 in proof 426 is not valid VM identity evidence. +- Rule: for nested SSH identity checks, run literal hostname commands without local command substitution. + +## FORUM_PROD_VM160_SWAP_PROOF_QUOTING_20260701 +- Context: VM160 swapfile proof 429. +- Issue: nested command substitution expanded on pve02, producing invalid SWAPFILE/FSTAB proof lines. +- Impact: proof 429 is not valid closure evidence even though swap was active. +- Rule: avoid nested $(...) in VM SSH proofs; use literal remote commands and clean proof. + +## FORUM_UPLOAD_ARCHIVES_VM_VERIFY_QUOTING_20260701 +- Context: proof 446 copy/check archives inside VM160. +- Issue: nested SSH quoting expanded shell variables incorrectly, producing gzip checks against empty .gz and blank TAR_TOP lines. +- Impact: SCP_TO_VM_RC=0 and file size list are valid, but GZIP/TAR proof lines in 446 are invalid. +- Rule: verify VM archive integrity locally inside forum-prod, not through nested quoted loops. + +## FORUM_PROD_BULK_IMPORT_PHP85_EMPTY_FRONTEND_20260701 +- Context: rebuilt VM160, five XenForo forums imported in bulk on PHP 8.5.7. +- Symptom: all five forums returned HTTP 200 with empty body; XenForo logged Template public:PAGE_CONTAINER/forum_list is unknown. +- Additional failed approach: proof 480 ran xf-dev:rebuild-caches on CodeVipe and returned RC=1, not a valid fix. +- Decision: stop bulk repair, treat current VM160 as lab, find a working CodeVipe recipe first, then destroy/rebuild cleanly for all five forums. +- Rule: no more mass rebuild/repair commands across all forums until single-forum CodeVipe recipe is proven. + +## XENFORO_REQUIREMENTS_DIRECT_DOWNLOAD_403_20260701 +- Context: Tried to download XenForo requirements ZIP directly from xenforo.com inside forum-prod. +- Issue: curl returned HTTP 403; unzip then failed because the ZIP was not downloaded. +- Evidence: proof 491 showed DOWNLOAD_RC=22, ZIP_LIST_RC=9, UNZIP_RC=9. +- Impact: proof 491 is not a valid server compatibility test. +- Rule: use operator-uploaded xenforo23-requirements-test.zip from /root/forum-upload instead of relying on direct curl download. + +## PVE01_UNZIP_MISSING_FOR_XF_REQUIREMENTS_VERIFY_20260701 +- Context: proof 492 tried to inspect uploaded xenforo23-requirements-test.zip on pve01. +- Issue: pve01 does not have unzip installed, so ZIP_TEST_RC=127 was not a ZIP integrity result. +- Impact: proof 492 confirmed file presence only, not archive validity. +- Rule: verify ZIP with Python zipfile or inside forum-prod rather than installing unzip on Proxmox just for inspection. + +## FRESH5_DEPLOY_SUCCESS_20260701 +- Context: VM160 was destroyed/recreated, then five XenForo forums were deployed from fresh ZIP backups. +- Result: proof 513 confirms all five forums locally healthy. +- Note: proof 511 initially failed dsmods path detection; proof 512 fixed dsmods by locating public_html via src/XF.php marker. +- Rule: future restores should preserve internal_data/code_cache or rebuild compiled template cache before smoke testing. + +## NPMPLUS_SQLITE_PASTE_FAILURE_20260701 +- Context: NPMplus SQLite inspection command was too complex and a Python fragment was pasted into bash. +- Issue: shell entered multiline prompt and produced syntax errors. +- Impact: do not trust that SQLite inspection attempt. +- Rule: avoid nested SSH heredoc/Python for this task; use simpler commands, generated nginx configs, or uploaded/local script files. + +## NPMPLUS_PUBLISH_SCRIPT_WRONG_ENV_SOURCE_525_20260701 +- Context: proof 525 tried to create forum proxy hosts in NPMplus. +- Issue: script sourced /opt/npmplus/.env, but INITIAL_ADMIN_EMAIL/PASSWORD were not present there; they are in the npmplus container env. +- Impact: no forum proxy hosts were created by proof 525. +- Rule: read NPMplus API login values from docker inspect env internally, never print them. + +## NPMPLUS_API_LOGIN_MISSED_HTTPS81_AND_JSON_ESCAPE_20260701 +- Context: NPMplus API login attempts in proofs 526/527 failed. +- Issue: commands missed the likely HTTPS admin endpoint on port 81 and built JSON by string interpolation instead of jq escaping. +- Impact: no proxy hosts were created by 526/527. +- Rule: for NPMplus API use jq-generated JSON and test https://127.0.0.1:81/api/tokens with -k before publish. + +## NPMPLUS_API_AUTH_UNAVAILABLE_MANUAL_ROUTES_20260701 +- Context: NPMplus API login attempts failed even though the forum VM and edge-to-forum connectivity are healthy. +- Issue: API credentials from container initial env are not accepted by current NPMplus API. +- Impact: do not use NPMplus API for this publish path. +- Rule: publish forum routes through backed-up manual nginx proxy_host configs, then issue proper certificates after DNS points to edge. + +## EDGE_CERTBOT_TOKEN_FILE_PERMISSION_535_20260701 +- Context: proof 535 attempted DNS-01 certificate issue for five forum domains. +- Issue: /tmp/forum_cf_token.env was root-owned mode 600, so debian user could not source it directly. +- Impact: certbot ran without token and all certificate attempts failed; DNS cutover must not proceed from proof 535. +- Rule: read Cloudflare token via sudo without printing it, then create temporary certbot credentials file and remove it after use. + +## EDGE_CERTBOT_FAILED_DNS_CUTOVER_ABORTED_20260701 +- Context: edge manual routes are working locally, but certbot DNS-01 certificate issue failed in proof 538. +- Issue: all five certbot attempts returned RC=1; proof 539 correctly aborted DNS cutover because edge cert precheck was not ready. +- Impact: Cloudflare DNS records were not changed by proof 539; public DNS still points to old IP 87.236.18.45. +- Rule: inspect certbot logs and Cloudflare token zone permissions before retrying certificate issuance or DNS cutover. + +## CLOUDFLARE_TOKEN_NO_ZONE_ACCESS_20260701 +- Context: Cloudflare token discovered on edge verified as a token but could not access forum zones. +- Evidence: proof 542 showed all five zones missing and DNS record create probes failed. +- Impact: certbot DNS-01 failed and DNS cutover was correctly aborted; public DNS still points to old hosting. +- Rule: install a new Cloudflare token with Zone Read and DNS Edit for the five forum zones before retrying certs/DNS. + +## FORUM_PUBLICATION_FINAL_SUCCESS_20260701 +- Context: five XenForo forums were rebuilt on VM160, routed through edge NPMplus, issued Let’s Encrypt certs using a corrected Cloudflare token, and cut over in DNS. +- Result: final public proof 546 passed. +- Important corrections: initial NPMplus API route failed; manual Nginx routes were used. Initial Cloudflare token had no zone access; corrected token passed zone/DNS probe. + +## FINAL_SNAPSHOT_NAME_TOO_LONG_556_20260701 +- Context: proof 556 final health gate passed for all five public forums. +- Issue: Proxmox snapshot failed because generated snapshot name exceeded 40 characters. +- Evidence: qm snapshot returned snapname value may only be 40 characters long. +- Impact: forum health was OK, but proof 556 snapshot step was not completed. +- Fix: rerun snapshot with short name. + +## CLOUDFLARE_DNS_AUDIT_TXT_SUBSTRING_GAP_20260701 +- Context: prior Cloudflare DNS audit checked web A/AAAA/CNAME state for root/www records. +- Issue: it did not deeply inspect TXT/SPF contents for old provider IP substrings. +- Evidence: user screenshot showed SPF TXT containing old 87.236.18.* value. +- Impact: web routing is OK, but mail-related DNS may still contain stale provider data. +- Rule: perform full DNS record audit including TXT/SPF/DKIM/DMARC/MX/CDN and substring search for old IPs before DNS cleanup. + +## CLOUDFLARE_DNS_CLEANUP_DRYRUN_QUOTE_563_20260701 +- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt. +- Impact: proof 563 is invalid and no DNS cleanup was performed by it. +- Rule: use copied scripts for Cloudflare DNS mutation instead of fragile inline nested quotes. + +## SPF_DUPLICATE_AFTER_565_20260701 +- Context: SPF cleanup command 565 attempted to replace stale SPF records. +- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained. +- Impact: domains temporarily had duplicate SPF records, which is invalid for mail validation. +- Fix: delete all SPF TXT records for the five forum zones, then create exactly one v=spf1 -all TXT per zone. + +## MSMTP_SECRET_SOURCE_PARSE_ERROR_576_20260701 +- Context: proof 576 installed msmtp but sendmail auth test failed. +- Issue: [SENSITIVE_PATH] was sourced as shell, but SMTP password contained shell-special characters; source failed and auth used an invalid/truncated secret path. +- Impact: msmtp package installed, but mail sending was not proven working. +- Rule: never source SMTP secret files containing arbitrary passwords; store password base64 and use msmtp passwordeval helper accessible to www-data. + +## FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701 +- Context: attempted to fix msmtp config with passwordeval helper. +- Issue: one or more sendmail/PHP mail tests still failed. +- Impact: XenForo mail sending is not yet proven. +- Next step: verify SMTP credential/password and provider policy for noreply@pvepro.ru. + +## SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701 +- Context: SMTP password was exposed in terminal output during failed msmtp setup. +- Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line. +- Impact: treat that SMTP password as compromised. +- Required action: rotate the mailbox/app password in the mail provider panel before any further SMTP testing. +- Rule: never store arbitrary passwords as shell-sourceable KEY=value files; use non-printing secret storage only. + +## NETBIRD_MAILCOW_REPORTED_DOWN_AFTER_SMTP_INCIDENT_20260701 +- Context: user reported NetBird and Mailcow became unavailable after forum mail-transport work. +- Impact: treat as active incident until service reachability and container/VM state are proven. +- Rule: do not change DNS, firewall, proxy, mail, NetBird, or Mailcow configs before read-only triage proof. + +## FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701 +- Context: forum-prod SMTP/msmtp test used invalid or compromised credentials and triggered Mailcow netfilter warnings/ban for 95.84.154.183. +- Impact: forum mail transport is not proven and must remain disabled until SMTP password is rotated and tested once carefully. +- Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth. +- Rule: do not test SMTP auth again until a fresh rotated mailbox/app password is available. + +## FORUM_OLD_CODEVIPE_CLOUD_BACKUP_STALE_AFTER_FRESH5_20260701 +- Context: pve02 codevipe-cloud-backup.timer existed before fresh five-forum rebuild. +- Issue: old script points to /root/backups/codevipe and old remote path app/codevipe/forum-prod/local. +- Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory contains old codevipe-public/codevipe.sql files, not current five-forum archive. +- Impact: old timer must not be treated as valid current backup for all five forums. +- Fix: create a new forum-prod fresh5 cloud backup using existing pve02 Mail-cloud rclone crypt remotes. + +## FORUM_RESTORE_DRILL_610_STATUS_FLAG_BUG_20260701 +- Context: restore drill proof 610 downloaded Mail.ru Cloud backup, verified split part SHA256, reconstructed archive, matched full archive SHA256, and confirmed all five forum DB/file archives in tar. +- Issue: final RESTORE_DRILL_OK was set to no because the script tested an unassigned shell variable RCLONE_REMOTE_PVE02_MAIL_01_RC. +- Impact: proof 610 final status flag is invalid, but its detailed integrity checks are valid. +- Fix: rerun corrected restore validation against the downloaded cloud backup artifacts and produce proof 612. + +## FORUM_XENFORO_MAIL_SMOKE_TEST_FAILED_20260701 +- XenForo-level mail smoke test did not return success for all five forums. +- Check proof 623 and msmtp log before retrying. + +## XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701 +- Context: custom XenForo mail smoke test 623 failed with Could not open input file /root/scripts/623_xenforo_mail_smoke.php. +- Cause: the script was run as www-data from /root/scripts; /root is not traversable by www-data. +- Impact: proof 623 is invalid and should not be used to judge mail delivery. +- Superseding evidence: user ran XenForo built-in outgoing email test; message was delivered to Yandex but placed in Spam. +- Next step: investigate deliverability/authentication headers and pvepro.ru SPF/DKIM/DMARC/PTR. + +## FORUM_NEXT_CHAT_KNOWN_ERRORS_AND_CLOSED_INCIDENTS_20260701 + +### Closed / classified incidents +- SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701: + - A previous bad command sourced a raw SMTP secret and printed it. + - Treat old password as compromised. + - Later persistent SMTP was rebuilt using safe files and verified. + - Never print or package secrets. + +- XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701: + - Custom mail proof 623 failed with "Could not open input file". + - Cause: PHP was run as www-data from /root/scripts, but /root is not traversable by www-data. + - Impact: proof 623 is invalid and must not be used to judge mail delivery. + - Superseded by user-observed built-in XenForo test and Yandex header proof. + +- FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701 / FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701: + - Earlier SMTP attempts failed and triggered mailcow netfilter ban for forum public IP. + - Ban was removed. + - Persistent msmtp transport was later enabled and verified. + - Mailcow and NetBird remained reachable after final tests. + +- SPF_DUPLICATE_AFTER_565_20260701: + - Earlier SPF cleanup created duplicate SPF records. + - Fixed by deleting duplicates and recreating exactly one SPF per forum domain. + - Final forum domains use v=spf1 -all because sending uses noreply@pvepro.ru. + +- FORUM_RESTORE_DRILL_610_STATUS_FLAG_BUG_20260701: + - Restore drill 610 had a status flag bug despite successful detailed checks. + - Corrected restore drill 612 passed. + +- FORUM_DBTECH_TOR_TIMEOUT_CLASSIFIED_20260701: + - One historical timeout row per forum to check.torproject.org. + - DBTech/Security active, but dbtech_security_tornodes=0. + - Classified as external timeout noise, not runtime failure. + +### Current non-blocking items +- Yandex placed built-in XenForo test mail in Spam despite SPF pass and DKIM pass. +- Classification: deliverability/reputation/content filtering, not server failure. +- Recommended manual action: click “Не спам!” and add/check pvepro.ru in Yandex Postmaster. + +### Safety rules for next chat +- Do not print secrets. +- Do not download or upload: + - [SENSITIVE_PATH] + - /etc/msmtprc + - /etc/msmtp/* + - rclone configs + - Cloudflare tokens + - DB dumps + - VM disks + - backup archives +- For handoff, only share the two truth files and selected non-secret proof files. + +## PARKED_DOMAINS_STAGE4_DNS01_PREFLIGHT_FAILED_20260701 +- Context: parked-domain public apply proof 634. +- Issue: DNS-01 based apply was attempted before a clear CAN_APPLY=yes preflight result was visible in terminal output. +- Observed: stage4 stopped safely with preflight_failed; CERTBOT_PRESENT=no, CERTBOT_DNS_CLOUDFLARE_PLUGIN=no, and CF_ZONE_ACCESS=no for newfi.ru, hapusya.ru and kingofwolk.ru. +- Impact: no parked-domain NPMplus route/certificate changes were applied by stage4; public HTTPS validation still failed certificate verification. +- Rule: do not use Cloudflare DNS-01 for these parked domains unless a token with explicit zone access is installed; because DNS already points to edge, prefer a copied-script HTTP-01 flow with temporary ACME challenge routes and content-specific validation. + +## PARKED_DOMAINS_STAGE5_BASH_LOCAL_SETU_BUG_20260701 +- Context: parked-domain HTTP-01 apply proof 635. +- Issue: edge script used Bash `local id="$1" ... conf="$WORK/.../$id.conf"` and `local host="$1" ... tmp="$WORK/.../$host.html"` under `set -u`; dependent variables are not safe inside the same local assignment command. +- Observed: `id: unbound variable`, `source can not be empty`, then `host: unbound variable`; certbot did not run, final HTTPS routes were not installed. +- Impact: local parked page remained OK; public HTTPS remained not closed. +- Rule: split dependent Bash local assignments into separate statements, syntax-check copied scripts before execution, and keep content-specific validation for parked routes. + +## PARKED_DOMAINS_STAGE6_TEMP_HTTP_ROOT_VALIDATION_BUG_20260701 +- Context: parked-domain HTTP-01 fixed apply proof 636. +- Issue: temporary HTTP-01 route validation checked HTTP `/` for parked marker, but expected behavior before certificates is root HTTP 301 to HTTPS. +- Observed: LOCAL_HTTP_VALIDATE returned HTTP_CODE=301 MARKER_OK=no, script rolled back its temporary route configs and did not run certbot. +- Impact: local parked page remained OK; public HTTPS remained not closed. +- Rule: validate `/.well-known/acme-challenge/` for HTTP-01 readiness, not HTTP `/`; also scan all active NPMplus nginx config dirs for unmanaged domain conflicts before writing routes. + +## PARKED_DOMAINS_STAGE7_NEWFI_ROOT_ACME_REDIRECT_20260701 +- Context: parked domains HTTP-01 Stage7 stopped before certbot because ACME validation path returned 301 for root newfi.ru while www.newfi.ru, hapusya.ru, www.hapusya.ru, kingofwolk.ru and www.kingofwolk.ru returned 200. +- Impact: no final parked-domain HTTPS routes/certificates were applied by Stage7; temporary route configs were rolled back. +- Rule: do not run another apply stage until active NPMplus nginx route selection for newfi.ru is diagnosed with nginx -T and Host-header probes. + +## PARKED_DOMAINS_STAGE8_DEFAULT_HTTP_TO_HTTPS_REDIRECT_20260701 +- Context: parked-domain route autopsy proof 638. +- Finding: no active NPMplus config contains newfi.ru/hapusya.ru/kingofwolk.ru, but unknown HTTP hosts and ACME paths return 301 to HTTPS from NPMplus default routing. +- Impact: HTTP-01 cannot work through the default server. A dedicated temporary ACME server block must be proven before certbot; do not run another certbot apply without a successful ACME-path probe for all six hostnames. +- Rule: after every parked-domain temp route attempt, rollback the route and prove rollback before continuing. + +## PARKED_DOMAINS_STAGE9_NEWFI_EXACT_PROBE_NEEDED_20260701 +- Context: parked domains HTTP-01 probe stage9 installed one combined temporary server block for all six hostnames. +- Issue: five hostnames returned ACME probe 200, but root newfi.ru still returned 301. +- Impact: do not continue certbot/apply until exact newfi.ru route behavior is isolated. +- Rule: run a temporary exact newfi.ru probe block and roll it back before any HTTP-01 apply. + +## PARKED_DOMAINS_STAGE10_EXACT_NEWFI_PROBE_OK_20260701 +- Context: parked-domain Stage10 proof 640. +- Result: exact server block for newfi.ru on NPMplus intercepted both root and ACME paths with HTTP 200 and custom X-Parked-Probe markers. +- Stage9 combined server_name block passed 5/6 but failed bare newfi.ru, so final HTTP-01 cutover must use separate exact per-host server blocks. +- Rule: do not use one combined parked-domain server_name block for these domains; use exact host blocks and content-specific validation. + +## PARKED_DOMAINS_STAGE11_LOW_ID_EXACT_ROUTE_FAILED_20260701 +- Context: parked-domain Stage11 proof 641. +- Result: exact per-host route files 700-705 passed nginx -t but bare newfi.ru still hit HTTP->HTTPS redirect on ACME path. +- Earlier Stage10 proof showed exact newfi.ru route works when placed at high id 998. +- Hypothesis: file/include priority or NPMplus generated ordering makes low id 700 unsuitable for bare newfi.ru in this runtime. +- Rule: do not retry parked-domain HTTP-01 with low id 700-705. First prove fixed high-id exact blocks, then apply with those exact ids. + +## PARKED_DOMAINS_SWITCH_TO_DNS01_AFTER_HTTP01_FAILURES_20260701 +- Context: parked domains newfi.ru, hapusya.ru and kingofwolk.ru. +- HTTP-01 attempts hit NPMplus default HTTP-to-HTTPS redirect/include-order problems. +- Stage12 cleanup confirmed temp files 980-985 absent and nginx reload OK. +- Rule: do not continue HTTP-01 for these parked domains; use DNS-01 with a dedicated Cloudflare token that has Zone Read and DNS Edit for the three zones. + +## PARKED_DOMAINS_STAGE15_FALSE_LOCAL_VALIDATE_ROLLBACK_20260701 +- Context: Stage15 DNS-01 succeeded and public validation showed parked page for most hosts, but local edge curl --resolve validation returned CURL_RC=60 and triggered rollback. +- Impact: do not rerun Stage15 as-is. +- Rule: for parked domains finalization, use existing DNS-01 certificates and validate from pve01 public HTTPS with content marker; do not use the failed local --resolve TLS check as the closure gate. + +## DOMAIN_ACTIONS_STAGE18_PLACEHOLDER_AND_RENEWAL_PROOF_20260701 +- Context: after parked domains and portfolio audit, operator requested removing domain enumeration from placeholder and checking certificate renewal paths. +- Rule: Stage18 must not change DNS records or NPMplus route files. It may update placeholder HTML, run existing forum renewal script once for runtime proof, and run certbot dry-run for parked certificates using existing DNS-01 hooks. + +## GRAM1_ROOT_WWW_PLACEHOLDER_CLOSE_STAGE19_20260701 +- Context: operator requested gram1.ru root/www to use the existing placeholder page. +- Rule: only root gram1.ru and www.gram1.ru may be routed. Existing gram1.ru subdomain NPMplus routes must not be changed. +- Rule: use DNS-01 with a dedicated gram1 Cloudflare token; do not use HTTP-01. +- Stage18 parked dry-run note: kingofwolk.ru dry-run hit Let's Encrypt rateLimited/service-busy after active certificate issuance and public HTTPS were already OK. Treat as transient external dry-run failure, not route/cert closure blocker. + +## PVEPRO_EDGE_LANDING_STAGE21_20260701 +- Context: pvepro.ru root/www currently point to the Mailcow VPS and return Mailcow with TLS hostname mismatch. +- Decision: because pve01 has no SSH key access to the external VPS, do not modify the VPS. Instead move only pvepro.ru and www.pvepro.ru A records to edge and serve an edge landing page. +- Safety rule: do not change mail.pvepro.ru, nb.pvepro.ru, MX, SPF, DKIM, DMARC or forum SMTP configuration. +- Use a dedicated Cloudflare token for pvepro.ru. Do not print token values. + +## PVEPRO_STAGE21_LANDING_SERVICE_FAILED_20260701 +- Context: Stage21 stopped safely before DNS/cert/NPMplus route because local pvepro landing service returned CURL_RC=7 on 127.0.0.1:18089. +- Impact: DNS root/www pvepro.ru remained on 185.139.214.215 and no route/cert changes were applied. +- Rule: Stage22 must fix and prove the local landing service first, then proceed to DNS-01 cert, NPMplus route and root/www DNS only after local marker PVEPRO_LANDING_OK is reachable. + +## PVEPRO_STAGE22_VALIDATION_AND_COPY_GUARD_20260701 +- Context: Stage22 succeeded on edge-side landing, cert, NPMplus route and Cloudflare A upserts, but wrapper had a non-critical scp wildcard failure due __pycache__ and public validation used cached DNS for root pvepro.ru. +- Mistakes recorded: do not scp scripts/* after local py_compile because __pycache__ may be present; do not continue after critical scp failure in future apply stages; do not classify PVEPro landing as NetBird just because it contains a link text; use Cloudflare authoritative A records and --resolve validation when DNS propagation may lag. +- Stage23 must be read-only for infrastructure and may close only if edge route, Cloudflare A records, HTTPS landing marker, and mail/nb unchanged checks pass. + +## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701 +- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router. +- Current limitation: there is no safe private remote access path to the dacha router yet. +- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet. +- Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists. + +## TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701 +- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router. +- Current limitation: there is no safe private remote access path to the dacha router yet. +- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet. +- Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists. + +## TAFTAUTO_WG_STAGE24_SECRET_AND_IMPORT_RULE_20260701 +- Context: configuring private management path for taftauto.ru dacha router. +- Rule: do not print WireGuard private keys, PSK, or client config body in proof/chat. +- Rule: generated client config may be stored only as root/debian mode 600 file and must be imported into the Netcraze router UI manually. +- Rule: do not open router public admin or certificate deployment until WireGuard handshake is proven. + +## TAFTAUTO_WG_PSK_ROTATION_SCRIPT_BROKE_DATAPLANE_20260702 +- Assistant gave unsafe PSK rotation flow; after profile reimport dacha interface changed from Wireguard0 to Wireguard1 and data-plane broke until security-level, ACL and routes were restored on Wireguard1. +- Proof 666 is superseded because it showed PUBLIC_SSH_22_STILL_OPEN. +- Final fixed proof: /root/evidence/667_TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702_PROOF.txt + +## TAFTAUTO_CERTBOT_CLOUDFLARE_PLUGIN_MISSING_20260702 +- Certbot on edge rejected --dns-cloudflare-* arguments: dns-cloudflare plugin missing/not loaded. +- No taftauto certificate issued in failed run; no DNS A records or NPMplus routes changed. + +## HOMEPAGE_SERVICES_YAML_BAD_INDENT_20260702 +- Assistant inserted Homepage services.yaml entries with wrong indentation; YAMLException bad indentation at line 227. +- Fix: rebuild SmartApe under External Homelabs with 2-space item indentation, remove Router Moscow, rebuild Sites category. + +## HOMEPAGE_SERVICES_YAML_BAD_INDENT_REPAIR_20260702 +- Previous apply broke services.yaml indentation and did not follow YAML-aware rule. +- Repair uses latest valid services.yaml backup, PyYAML parse/dump, validates before restart. + +## TAFTAUTO_CERTBOT_DRYRUN_BAD_FLAG_20260702 +- Assistant used unsupported certbot flag --manual-public-ip-logging-ok with certbot 4.0.0 during taftauto renew dry-run. +- No certificate, DNS, NPMplus route, or deploy state changed by the failed dry-run. + +## TAFTAUTO_CERT_RENEW_DRYRUN_RATE_LIMITED_20260702 +- Proof 672 is superseded/partial: it printed STATUS=OK even though certbot dry-run failed. +- Failure reason: Let's Encrypt staging/service returned rateLimited / Service busy; retry later. +- Cloudflare manual auth and cleanup hooks did run successfully. +- Deploy hook was installed and manually invoked successfully before the dry-run. +- Do not retry immediately. + +## HOMEPAGE_XENFORO_INFO_INLINE_PYTHON_SYNTAX_ERROR_20260702 +- Assistant used invalid inline Python syntax with def after semicolon while adding XenForo.info to Homepage External. +- Failed before services.yaml write. + +## HOMEPAGE_USEFUL_ROUTER_NAME_MISMATCH_20260702 +- Previous apply looked for Russian Router card name, but active Homepage Useful group contains Router and Public Domain. +- Failed before write. Fix uses YAML-aware edit with exact names Router/Public Domain. + +## HOMEPAGE_USEFUL_ROUTER_CARD_NOT_FOUND_20260702 +- Apply expected Useful -> Router in active services.yaml, but YAML parse did not find it. No write was performed. +- Need verify active Homepage config path, docker mount, and any alternate services.yaml before next apply. + +## 20260702_CF_TOKEN_AUDIT_BROKEN_COMMAND +- Ошибка: ассистент дал хрупкую команду аудита Cloudflare-токенов с вложенным heredoc/Python через ssh. +- Факт: bash начал интерпретировать Python-код, появились syntax error и вторичный prompt >. +- Правило: не давать nested heredoc/сложные кавычки через ssh; для длинной логики сначала класть скрипт отдельным файлом, затем запускать его. +- Статус: аудит токенов не выполнен, инфраструктурные изменения не должны считаться применёнными. + +## LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE +- Do not treat e3qxxx.netbird.selfhosted / 100.100.125.70 and e3qxxx-183-106.netbird.selfhosted / 100.100.183.106 as VPS egress nodes; they are user mobile peers. +- Correct egress candidates are relay.pvepro.ru / relay.netbird.selfhosted / 100.100.19.1 for Moldova and mail.pvepro.ru / mail.netbird.selfhosted / 100.100.147.204 for USA. +- Access Policy edge-vm -> relay/mail only proves peer reachability. It is not an internet exit route. +- Required NetBird fix for CrowdSec CAPI was Network Routing exit route 0.0.0.0/0 distributed to edge-vm group, routing peer relay, masquerade/NAT enabled. +- Before attempting CrowdSec CAPI, verify on edge-vm: ip route get 1.1.1.1 uses wt0, wg allowed-ips contains 0.0.0.0/0, and Cloudflare trace no longer shows home IP 95.84.154.183. + +## LESSON_20260702_HOMEPAGE_MONITORS_REPAIR_NOT_DELETE +- Do not delete or disable Homepage siteMonitor fields to hide red badges. +- Fix the underlying monitor path, DNS, proxy route, or local health-wrapper instead. +- Do not touch Cloudflare Homepage card when operator says it is green and opens correctly. +- Before changing Homepage, identify the live config host; in this state live Homepage is on core-apps [PRIVATE_IP], not edge-vm. + +## LESSON_20260702_KUMA_ADD_ONE_MONITOR_ONLY +- When asked to fix one missing Uptime Kuma service, add only one monitor, then generate proposals separately. +- Before direct Kuma DB mutation, stop the container and create a DB backup. +- Verify DB integrity before starting Kuma again. +- Do not touch Cloudflare when operator says it is green and opens correctly. + +## LESSON_20260702_DOCKGE_STALE_STACKS_NOT_CONTAINERS +- Dockge inactive items after migration can be stale compose folders, not stopped containers. +- First classify runtime projects across all Docker hosts before deleting or archiving anything. +- For remote Dockge visibility, projects must be under /opt/stacks or bind-mounted there. +- Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions. + +## STAGE4B_MASKED_SQL_QUERY_FAILURE_20260714 +- Previous Stage4B read-only audit returned empty observation-summary, metadata-key, trigger and function blocks because nested SSH/SQL quoting broke queries while stderr was hidden. +- Empty blocks are query failures, not proof that metadata, triggers or functions are absent. +- Future PostgreSQL audits must use ON_ERROR_STOP, visible stderr, independent query RC and quote-safe dollar literals. + +## STAGE4C_SCHEMA_MIGRATIONS_ID_ASSUMPTION_20260714 +- Stage4C read-only preflight incorrectly assumed schema_migrations has a column named id. +- PostgreSQL returned column id does not exist and ON_ERROR_STOP correctly stopped all following SQL checks. +- Production DB and application were not changed. Future migration audits must inspect information_schema first and query migration rows without assuming column names. + +## STAGE4C_PGDUMP_DEV_NULL_FSYNC_INVALID_PROBE_20260714 +- Stage4C custom-format pg_dump capability probe incorrectly used /dev/null as the output file. +- pg_dump failed only because fsync on /dev/null returned Invalid argument. +- Production database and application were not changed. +- Future custom dump capability checks must use a real temporary regular file, validate it with pg_restore --list, and remove it afterward. + +## STAGE4C_REMOTE_PREP_THIRD_SUBSTEP_FAILED_20260714 +- Isolated migration dry-run did not start: REMOTE_PREP_RC=0|0|1. +- Remote directory creation and SCP succeeded; the third ownership/mode preparation substep returned RC=1. +- Production database and application were not changed. +- Exact cause requires read-only residual directory and permission inspection before cleanup or retry. + +## STAGE4C_REMOTE_PREP_GLOB_AFTER_CHMOD_ROOT_CAUSE_20260714 +- Exact cause of REMOTE_PREP_RC=0|0|1: after the directory became postgres-owned mode 0700, the unprivileged debian shell could not traverse it and could not expand the wildcard used by chmod. +- Candidate files and SHA256 verification were valid; no temporary database was created and production was unchanged. +- Future preparation must use exact privileged paths or a verified shared group, never an unprivileged wildcard after restrictive chmod. + +## STAGE4C_MIGRATION003_FINGERPRINT_DOLLAR_QUOTE_FAILURE_20260714 +- Migration 003 isolated dry-run failed at fingerprint CHECK because the regex end anchor collided with the SQL dollar-quote delimiter and produced an unterminated dollar-quoted string. +- The migration transaction rolled back, the temporary database was removed, and production remained unchanged. +- Fingerprint validation must use length plus translate without a regex dollar anchor; corrected candidate must pass a fresh isolated dry-run before production consideration. + +## STAGE4C_SEAL_OUTER_RC_MASKING_20260714 +- Stage4C final seal correctly refused creation because overall health was WARN, returning internal RC=40. +- The former wrapper then printed a second misleading COMMAND_RC=0 because top-level true masked the internal result. +- Future commands must capture the guarded subshell through PIPESTATUS and print exactly one authoritative COMMAND_RC before the mandatory final true. +- Production database, application and services were unchanged. + +## STAGE4C_DIAGNOSIS_EXPECTED_DERIVED_FILE_ON_VM180_20260714 +- Previous diagnosis incorrectly treated cluster-admin-restricted-probes.txt as a required source file on VM180. +- The file is a central derived health artifact on pve01; VM180 supplies full-observer evidence instead. +- This false expectation caused diagnostic RC=32 without any infrastructure failure or production change. + +## STAGE4C_NO_JOBS_RUNNING_TEXT_COUNTED_AS_JOB_20260714 +- Stage4C seal preflight incorrectly counted the systemctl informational text No jobs running as one active job. +- The parser counted every nonempty output line instead of accepting only rows beginning with a numeric systemd Job ID. +- No service was started and no production state changed during the blocked attempt. +- Future job counts must match a numeric first field only. + +## STAGE4D_ADAPTER_SOURCE_AUDIT_ABORTED_BEFORE_FINAL_MARKERS_20260714 +- The evidence-only Stage4D adapter source audit returned RC=1 before emitting its VM180 completion and final audit markers. +- The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults. +- Production, database, application, services, timers, health and desired-state were unchanged. +- Inspect the preserved runner stdout, stderr and result JSON before correcting or rerunning the task. + +## STAGE4D_LOCAL_EXPANSION_OF_REMOTE_Q_UNDER_NOUNSET_20260714 +- The first Stage4D adapter source audit embedded a large remote program inside a locally double-quoted SSH argument. +- Local shell expansion reached the remote variable Q while nounset was enabled, causing Q: unbound variable before SSH execution. +- VM180 and PostgreSQL audits did not start; production state was unchanged. +- Remote shell and SQL programs must be separate syntax-checked files passed through SSH stdin. + +## STAGE4D_REMOTE_PYTHON311_ADAPTER_SELFTEST_RC1_20260714 +- Canonical adapter candidate passed local Python 3.13 self-tests with 48 envelopes but returned RC=1 on VM180 Python 3.11.2. +- Remote upload and cleanup succeeded, and production database remained 0|0|OK. +- The exact traceback and failing assertion must be captured by an isolated identical reproduction before changing the candidate. + +## STAGE4D_PYTHON311_FSTRING_COMPATIBILITY_AND_REPRO_VALIDATOR_20260714 +- Adapter candidate passed Python 3.13 but failed Python 3.11 parsing at Path(row["path"]) inside a double-quoted f-string. +- Exact fix is Python 3.11-compatible quoting: Path(row['path']). +- The reproduction command captured the expected compile failure but incorrectly returned CAPTURE_BAD=1 because its validator required compile RC=0. +- Production database remained 0|0|OK and desired-state remained clean. +- Future adapter candidates must pass syntax and full self-tests on both pve01 and VM180 runtimes before acceptance. + +## STAGE4E_DESIGN_VALIDATOR_ABORTED_BEFORE_LOCAL_VALIDATION_MARKER_20260714 +- Restricted probe-agent design candidate compiled successfully but its local validator exited before producing LOCAL_VALIDATION_RC. +- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults. +- Active external probes were not executed and production state was unchanged. +- Inspect the preserved validator traceback and exact assertion before modifying the candidate. + +## STAGE4E_VALIDATOR_SCANNED_PYC_AND_OWN_FORBIDDEN_LITERALS_20260714 +- Stage4E design validator recursively read every candidate file as UTF-8 after py_compile had created binary __pycache__ bytecode. +- This caused UnicodeDecodeError before any design assertion failed. +- The phase-wide forbidden-pattern scan also included validate_design.py itself, which necessarily contains the prohibited literals it verifies. +- Validators must use an explicit text-file allowlist and scans must exclude validator implementation and binary artifacts. +- Production, database, services, timers and desired-state were unchanged; active external probes were not executed. + +## STAGE4E_V2_VALIDATOR_FAILED_STATIC_COMPILE_20260714 +- Stage4E design candidate v2 failed static compilation before local design validation started. +- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects. +- The generated validator must be inspected at the exact SyntaxError line before another candidate is created. +- Production, database, application, services, timers, health and desired-state were unchanged; active external probes were not executed. + +## STAGE4E_PIPESTATUS_LOST_AFTER_FIRST_ASSIGNMENT_20260714 +- VM180 validation copy used a pipeline and read PIPESTATUS in two separate assignments. +- The first assignment reset PIPESTATUS, leaving COPY_RC empty and preventing remote validation and cleanup. +- Retry must use explicit SCP operations without pipeline status parsing. +- Production remained unchanged and active external probes were not executed. + +## STAGE4F_VALIDATOR_EXPECTED_HEALTH_REFRESH_FROM_NONWRITING_BACKUP_SCRIPT_20260714 +- The controlled backup service completed with Result=success and ExecMainStatus=0. +- Validation incorrectly required cluster-admin-incident-engine-backup.txt to receive a fresh timestamp. +- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure. +- Stage4F backup proof must use the service result, fresh latest JSON, produced artifacts and off-host/restore evidence instead. +- The backup must not be rerun merely to satisfy the invalid health-file freshness assertion. + +## STRICT_RULE_20260714_CLOSE_TAILS_IMMEDIATELY +- Failure class: переход к следующей задаче при наличии незакрытого хвоста. +- Mandatory anti-regression: после ошибки запрещено продолжать другой scope до точной root cause, исправления, повторной проверки, очистки, proof и seal. +- Closure gate: UNRESOLVED_TAIL_COUNT=0; BLOCKER_COUNT=0; TEMPORARY_ARTIFACT_COUNT=0; ROLLBACK_VERIFIED=YES; HEALTH=OK; PROOF_UPDATED=YES; REFERENCE_UPDATED=YES. +- Если закрытие невозможно из-за внешней зависимости, опасной неоднозначности или обязательного решения пользователя, фиксировать BLOCKED/OPEN и не заявлять CLOSED. +- Proof: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/report.txt + +## ERROR_20260714_DEPENDENCY_FACT_QUERY_GLOBAL_LIMIT +- Symptom: dependency audit вернул только latest_collector_status. +- Root cause: ORDER BY 1 LIMIT 1 в конце UNION ALL применился ко всему набору фактов. +- Correction: latest collector status извлекается скалярным подзапросом внутри CTE facts; глобальный LIMIT отсутствует. +- Anti-regression: multi-fact SQL audit обязан проверять точное ожидаемое число строк до извлечения значений. +- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt +- Status: CLOSED. + +## ERROR_20260714_COLLECTOR_PATCH_REQUIREMENT_PATTERN_MISMATCH +- Symptom: MIGRATION_PLAN_COLLECTOR_PATCH_REQUIRED_COUNT=1 при ожидаемых двух подтверждениях. +- Root cause: узкий шаблон учитывал collector_patch_required, но не collector_patch_required_after_stage4c. +- Correction: оба поля проверяются отдельно, затем складываются как semantic requirement count. +- Anti-regression: связанные JSON-контракты проверяются по собственным точным именам полей, а не одним унифицированным grep-шаблоном. +- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt +- Status: CLOSED; PRIOR_TAIL_UNRESOLVED_COUNT=0. + +## ERROR_20260714_PRIVILEGED_FILE_REDIRECTION_BEFORE_SUDO +- Symptom: bash reported Permission denied while counting collector.py lines. +- Root cause: `< collector.py` was opened by the unprivileged caller shell before sudo executed wc. +- Correction: run sudo wc -l collector.py without caller-side input redirection. +- Anti-regression: never combine sudo with `< protected-file`; pass the protected path as an argument to the privileged command. +- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt +- Status: CLOSED. + +## ERROR_20260714_COLLECTOR_RUN_COLUMN_ASSUMPTION +- Symptom: SQL failed because completed_at did not exist. +- Root cause: collector_runs columns were assumed instead of read from information_schema; actual names are finished_at and error_text. +- Correction: assert required and forbidden column counts before querying recent runs. +- Anti-regression: every schema-sensitive audit must discover and validate exact column names before the main query. +- Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. + +## ERROR_20260714_REMOTE_STDIN_ARGUMENT_SHIFT +- Symptom: remote harness выполнил chmod для пути bash. +- Root cause: дополнительный positional token bash был передан после SSH command и стал первым аргументом stdin-скрипта. +- Correction: remote command вызывает bash -s -- с ровно тремя явно встроенными аргументами: candidate path, SHA256 и byte count. +- Anti-regression: stdin-скрипт обязан валидировать $1 по разрешённому path pattern и затем подтверждать точные SHA256 и byte count; запрещено добавлять отдельный bash-token после remote command. +- Closure proof: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/report.txt +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. + +## ERROR_20260714_OVERSIZED_COMMAND_PARSE_FAILURE +- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`. +- Root cause: чрезмерно длинная интерактивная команда была обрезана или повреждена при передаче до полного разбора Bash. +- Impact: side-effect audit подтвердил production state 0|0|OK, неизменный collector, активный timer и отсутствие временных БД, каталогов и процессов. +- Correction: монолитные команды запрещены; сложные действия выполняются через отдельный проверяемый task-скрипт и короткий launcher. +- Anti-regression: interactive payload target <= 8000 bytes; syntax-check перед запуском; remote output сохраняется независимо от RC. +- Closure proof: /var/lib/homelab-change-sets/operator-command-size-guard-v1/20260714T182653Z/report.txt +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. + +## ERROR_20260714_HOMELAB_ADMIN_HELP_RC_ASSUMPTION +- Symptom: template audit завершился HOMELAB_ADMIN_HELP_FAILED. +- Root cause: был ошибочно ожидаем RC=0 от неподдерживаемого аргумента --help. +- Actual contract: неизвестный аргумент печатает usage и возвращает RC=64. +- Correction: проверять version либо считать точную usage-строку вместе с RC=64 штатным результатом. +- Anti-regression: перед проверкой CLI сначала фиксировать поддерживаемые команды; не предполагать GNU-style --help. +- Closure proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v1/20260714T183338Z/report.txt +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. + +## ERROR_20260715_EXPECTED_NEGATIVE_RC_TRAP_CLASSIFICATION +- Symptom: schema-v2 task v3 build stopped while intentionally testing the --invalid argument path. +- Root cause: an expected RC64 was executed while the generic ERR trap remained active. +- Correction: expected failures are evaluated only through an explicit if/else branch with immediate RC capture. +ANTI_REGRESSION_TOKEN=EXPECTED_RC64_INTERCEPTED_ERR_TRAP +- Required pattern: if command; then rc=0; else rc=$?; fi, followed by a separate exact expected-RC assertion. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. + +## ERROR_20260715_DUPLICATE_MACHINE_MARKER_COUNT +- Symptom: error-register candidate construction stopped with RC1 before applying the candidate. +- Root cause: the same marker text appeared in both a heading and a descriptive line, while the guard expected one substring occurrence. +- Correction: headings and prose do not contain machine-token values; validation counts only complete exact token lines. +ANTI_REGRESSION_TOKEN=DUPLICATE_MACHINE_TOKEN_IN_HEADING_AND_BODY +- Required pattern: grep -Fxc against a complete machine line, never grep -Foc against an unrestricted substring. +- Production impact: none. +- Temporary artifacts: removed and verified. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T02:56:19Z + +## ERROR_20260715_OUTER_WRAPPER_SHELL_SYNTAX +- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis. +- Root cause: the outer wrapper contained an unbalanced compound shell construct before the closing subshell delimiter. +- Impact: parsing failed before the start marker, guards and homelab-admin invocation; no task phase or infrastructure action executed. +- Correction: use a linear wrapper with explicitly closed if/case blocks and avoid nested brace groups in conditional expressions. +ANTI_REGRESSION_TOKEN=OUTER_WRAPPER_UNBALANCED_SUBSHELL_SYNTAX +- Anti-regression: after any parser-level failure, treat the attempted phase as not executed and verify package and run-state before retrying. +- Production impact: none. +- Task package impact: none. +- Temporary artifacts: none. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T03:24:05Z + +## ERROR_20260715_INLINE_REMOTE_SHELL_IN_APPLY_PHASE +- Symptom: homelab-admin lint rejected Stage4H task v6 with ERR-002-INLINE-REMOTE-SHELL. +- Root cause: apply.sh invoked bash -s through an inline quoted SSH program and streamed the remote script through stdin. +- Correction: copy the declared remote script to an isolated fixed path on VM180 and invoke that file directly with positional arguments. +ANTI_REGRESSION_TOKEN=INLINE_REMOTE_SHELL_REPLACED_BY_STAGED_REMOTE_SCRIPT_PATH +- Anti-regression: phase scripts may perform transport orchestration, but remote shell program bodies must exist as separately declared, syntax-checked remote script files. +- Remote stdout and stderr must be preserved before evaluating the remote return code. +- Task v6 status: REJECTED_BY_LINT; never executed and never mutated. +- Production database impact: none. +- Live collector impact: none. +- Temporary database and remote root count after rejection: zero. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T04:13:46Z + +## ERROR_20260715_OPTIONAL_RUN_DIRECTORY_FIND_UNDER_PIPEFAIL +- Symptom: task-v7 builder stopped while counting formal runs for rejected task v6. +- Root cause: find was executed against an optional directory that did not exist; under pipefail the assignment returned RC1. +- Correction: test directory existence first and assign zero without invoking find when it is absent. +ANTI_REGRESSION_TOKEN=MISSING_OPTIONAL_RUN_DIRECTORY_TREATED_AS_ZERO +- Anti-regression: optional paths must have an explicit existence branch before find under pipefail. +- Production impact: none. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T04:44:06Z + +## ERROR_20260715_PG_CONSTRAINT_CONTYPE_CHAR_CONCAT +- Symptom: isolated Stage4H acceptance failed during schema baseline capture. +- Root cause: pg_constraint.contype uses the internal PostgreSQL char type and was concatenated without an explicit text cast. +- Correction: use contype::text or CAST(contype AS text). +ANTI_REGRESSION_TOKEN=PG_CATALOG_CHAR_CONCAT_REQUIRES_EXPLICIT_TEXT_CAST +- Machine rule: ERR-014-PG-CATALOG-CHAR-CONCAT. +- Negative self-test: uncast expression rejected with RC64. +- Positive self-test: explicit text cast accepted with RC0. +- Task v7 mutated: no. +- Production impact: none. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T06:27:06Z + +## ERROR_20260715_LIVE_COLLECTOR_VERIFIER_WRONG_PATH +- Symptom: verified linter installation was rolled back after the final collector integrity guard returned UNKNOWN. +- Root cause: verifier queried a nonexistent path instead of the canonical live collector path declared by the task acceptance script. +- Canonical path: /opt/cluster-admin-incident-engine/collector.py. +- Incorrect path: /opt/homelab-cluster-admin/cluster-admin-collector.py. +ANTI_REGRESSION_TOKEN=LIVE_COLLECTOR_VERIFIER_PATH_MUST_MATCH_TASK_CANONICAL_PATH +- Correction: derive and guard the live collector path from the immutable task contract before executing the hash query. +- Production impact: none; the canonical collector hash remained unchanged. +- Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. +- Registered at: 2026-07-15T06:27:06Z + +AUTOREFRESH_INLINE_FAILURE_20260721 +Attempts 048 and 049 did not appear in immutable history. +Rule: use validated local helper scripts and a simple scheduler entry; never embed the full collector in cron or nested SSH quoting. + +AUTOREFRESH_WRAPPER_PUBLICATION_FAILURE_20260721 +Commands 044-051 did not reach immutable history through homelab-chat-run. +Resolution: hourly snapshots use direct scp plus homelab-runtime-receive publication. + +AUTOREFRESH_052_NOT_PUBLISHED_20260721 +CONTEXT-AUTOREFRESH-ACTIVATE-052 did not appear in immutable history; direct publisher retry requires sanitization before transmission. + +IMMUTABLE_HISTORY_FALSE_NEGATIVE_20260721 +BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative. +Rule: verify the exact COMMAND_ID result, wrapper publication status and commit SHA; never infer absence from a cached directory listing. + +BACKUP_UNSAFE_SOURCE_CAPTURE_071_PARSE_ERROR_20260721 +BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (. +Resolution: avoid nested quote layers; parse targets first and pass file paths as simple SSH arguments. +DETAIL=SECTION_ERROR_REGISTER_END +DETAIL=SECTION_STRUCTURED_ERRORS_INDEX_BEGIN +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "ERRORS-INDEX-004", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "error-ledger", + "started_at_utc": "2026-07-21T07:22:22Z", + "finished_at_utc": "2026-07-21T07:22:22Z", + "reference_register_checked": true, + "reference_sha256": "f6f7749fda946015116cdbbd754c7e7f20356b4a6cf4df330a1b0b8fdf59cbb2", + "error_register_checked": true, + "error_register_sha256": "ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc", + "command_sha256": "a45b24ab6a161508f31ee8a6b954c045e74824b4d12adfef6f298cfac6111016", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "655ec287a9095e799677cb7f87b08d4119cbdf9027d7097fa610da5a022ab30b", + "sanitized_output_sha256": "655ec287a9095e799677cb7f87b08d4119cbdf9027d7097fa610da5a022ab30b", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "{\n \"schema_version\": 1,\n \"status\": \"READY\",\n \"generated_at_utc\": \"2026-07-21T07:22:22.821762Z\",\n \"source\": {\n \"path\": \"/etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md\",\n \"sha256\": \"ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc\",\n \"line_count\": 934,\n \"sanitized\": true\n },\n \"summary\": {\n \"entry_count\": 154,\n \"rule_count\": 90,\n \"duplicate_entry_ids\": [],\n \"duplicate_entry_signatures\": [\n \"9cd06e064f5a86be9626c4f9a2a819b8d0e71b03309c705fe0e34e1e59f81b73\"\n ],\n \"duplicate_rule_signatures\": [\n \"b936ae30ffb935118585b5a133d9c32643946a3ba5034b85f42bae000518d49a\"\n ]\n },\n \"entries\": [\n {\n \"id\": \"ERR-H-L1-HOMELAB-ASSISTANT-ERROR-REGISTER\",\n \"kind\": \"heading\",\n \"level\": 1,\n \"source_line\": 1,\n \"title\": \"HOMELAB ASSISTANT ERROR REGISTER\",\n \"summary\": \"Назначение: перед каждой следующей командой сверяться с этим файлом.\",\n \"rule_like\": true,\n \"signature\": \"cd4b4d42c26702b03e3e81a620c46dbe3d498f3f2233215024a5c2a49cc1dbfc\"\n },\n {\n \"id\": \"ERR-N-1-L6\",\n \"kind\": \"numbered\",\n \"source_line\": 6,\n \"title\": \"Повторно дал слишком большой интерактивный paste в shell.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"f732778534f22b73e31dead215c9c7078f48a552a5341ae3830eea618885e8a7\"\n },\n {\n \"id\": \"ERR-N-2-L7\",\n \"kind\": \"numbered\",\n \"source_line\": 7,\n \"title\": \"Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"b13c8946e274927bd041960c76208b6f62e22d46f07233388878c1f47afc426a\"\n },\n {\n \"id\": \"ERR-N-3-L8\",\n \"kind\": \"numbered\",\n \"source_line\": 8,\n \"title\": \"Дал генератор справочника прямо в терминал вместо безопасного маленького шага.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"8de70b8d263233557a33a065ee49004ef01e328de0d736eb9e328930119458be\"\n },\n {\n \"id\": \"ERR-N-4-L9\",\n \"kind\": \"numbered\",\n \"source_line\": 9,\n \"title\": \"Нарушил своё же правило: не давать длинные вложенные команды с кавычками.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"d1bf3e42845e6c9d25511814c1da9b21cd8001474f17fd60240194f812f06fe0\"\n },\n {\n \"id\": \"ERR-H-L5-КРИТИЧЕСКИЕ-ОШИБКИ-АССИСТЕНТА\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 5,\n \"title\": \"Критические ошибки ассистента\",\n \"summary\": \"1. Повторно дал слишком большой интерактивный paste в shell. 2. Повторно использовал here-doc/Markdown/backticks там, где нужен короткий безопасный файл или отдельный скрипт. 3. Дал генератор справочника прямо в терминал вместо безопасного маленького шага. 4. Нарушил своё же правило: не давать длинные вложенные команды с кавычками.\",\n \"rule_like\": false,\n \"signature\": \"0ba065b54f515fe668143ec87d006ea8f27639e6529fe46367ac9acd6f2e28d6\"\n },\n {\n \"id\": \"ERR-H-L11-ЖЁСТКИЕ-ПРАВИЛА-ПЕРЕД-КАЖДОЙ-КОМАНДОЙ\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 11,\n \"title\": \"Жёсткие правила перед каждой командой\",\n \"summary\": \"CHECK-1: команда не должна быть большим paste. CHECK-2: команда не должна содержать большой here-doc. CHECK-3: команда не должна смешивать Markdown, backticks и shell-логику. CHECK-4: команда не должна иметь вложенный ssh с несколькими уровнями кавычек. CHECK-5: команда не должна печатать секреты. CHECK-6: если создаётся файл, сначала маленький безопасный шаг, потом проверка. CHECK-7: если команда длиннее 8 строк, её нельзя давать в интерактивный shell. CHECK-8: для edge-vm использовать debian@[PRIVATE_IP] и sudo.\",\n \"rule_like\": true,\n \"signature\": \"7ac085a9df3e592d1d7d6686ca4980278110a07e296ddc0253cedb075cc08d84\"\n },\n {\n \"id\": \"ERR-H-L23-ТЕКУЩИЕ-ВАЖНЫЕ-ФАКТЫ\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 23,\n \"title\": \"Текущие важные факты\",\n \"summary\": \"Internal network: [PRIVATE_IP]/24. Migration config: migration: secure,network=[PRIVATE_IP]/24. Corosync remains on [PRIVATE_IP]/12/13. VM160 forum-prod is not in Proxmox nightly backup. VM130 edge-vm has secondary disk backup=0 risk. 05_edge_compose_safe.tgz quarantined.\",\n \"rule_like\": false,\n \"signature\": \"ecd7f734df435c41c5f01c9fd7f9371a4012c6019493072741c45c6eaf3ed383\"\n },\n {\n \"id\": \"ERR-N-11-L35\",\n \"kind\": \"numbered\",\n \"source_line\": 35,\n \"title\": \"Ошибка: считать offhost OK после failed rsync.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"aa8a2a5cc3228d73d2feadd69fa2fb1d0f76d020280f69921ac5c00dcfc1f9fd\"\n },\n {\n \"id\": \"ERR-N-12-L40\",\n \"kind\": \"numbered\",\n \"source_line\": 40,\n \"title\": \"Ошибка: широкий secret-поиск по /opt/stacks дал шум.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"da33bb4eaab4ca4827c1e122c4292fa4181a7607db77ebce5d185b867bfe0ac3\"\n },\n {\n \"id\": \"ERR-N-13-L45\",\n \"kind\": \"numbered\",\n \"source_line\": 45,\n \"title\": \"Ошибка: SQL с одинарными кавычками внутри одинарной SSH-команды ломается.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"888cbc312835f6303e74db4c5ed2dcaef1e552e657f83ca6f3f82834eb06c23e\"\n },\n {\n \"id\": \"ERR-N-14-L50\",\n \"kind\": \"numbered\",\n \"source_line\": 50,\n \"title\": \"Ошибка: путать контекст входа и узел выполнения.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"97eec6b76b51ffbc78105ce4e450846c5ff1c85dfbcdfeb3e6c687addd6a1391\"\n },\n {\n \"id\": \"ERR-N-15-L57\",\n \"kind\": \"numbered\",\n \"source_line\": 57,\n \"title\": \"Ошибка: повторно нарушено правило №13 после его добавления.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"95160c6fda4d8bc77fc542236363b182ba66a7314a8032379b033a0feba5e3b2\"\n },\n {\n \"id\": \"ERR-N-16-L63\",\n \"kind\": \"numbered\",\n \"source_line\": 63,\n \"title\": \"Основополагающее правило: перед каждой командой ассистент обязан явно написать: Сверка с файлом ошибок: CHECK пройден.\",\n \"summary\": \"\",\n \"rule_like\": true,\n \"signature\": \"254db5ca3a2611269de7af81ececd43035134e01a8e629c34434913c49eeeb41\"\n },\n {\n \"id\": \"ERR-N-19-L68\",\n \"kind\": \"numbered\",\n \"source_line\": 68,\n \"title\": \"Уточнение формата: не писать отдельную строку сверки перед каждой командой в пакете.\",\n \"summary\": \"\",\n \"rule_like\": true,\n \"signature\": \"a77bda86805176e2dc721961454b31a5bff15fe924bd43496475c5816a8f9650\"\n },\n {\n \"id\": \"ERR-N-20-L73\",\n \"kind\": \"numbered\",\n \"source_line\": 73,\n \"title\": \"Ошибка: широкий XenForo SMTP-поиск по src дал шум исходников.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"bed26484549b8d47c75ee6150c206503c05074cc8a5b25dfde0735596aaf956c\"\n },\n {\n \"id\": \"ERR-N-21-L77\",\n \"kind\": \"numbered\",\n \"source_line\": 77,\n \"title\": \"Ошибка: nested PHP php -r дал Parse error на forum-prod.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"1198d8f0576c5dc8e3883ac7d7e423c8391e946c73bf5ac3cf6063278a68a5af\"\n },\n {\n \"id\": \"ERR-N-22-L82\",\n \"kind\": \"numbered\",\n \"source_line\": 82,\n \"title\": \"Ошибка: самодельный base64 PHP для SMTP auth сломан.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"c629522366fb27522bec6dd9b9b24f532bbc9c3372fbf4d706673bf283bdf498\"\n },\n {\n \"id\": \"ERR-N-23-L87\",\n \"kind\": \"numbered\",\n \"source_line\": 87,\n \"title\": \"Ошибка: MariaDB не поддержала cast('true' as json) в JSON_SET.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"4faccf6aa3103144040fd4efb62dd82e23d9f28ecfba72502293f6e2f060b819\"\n },\n {\n \"id\": \"ERR-N-24-L92\",\n \"kind\": \"numbered\",\n \"source_line\": 92,\n \"title\": \"Ошибка: exit 1 в interactive-check закрыл SSH-сессию.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"c522fbe5bc7aadfc904f1bde7328952b554884941afb3558e60cfe6a4ed378cd\"\n },\n {\n \"id\": \"ERR-N-25-L97\",\n \"kind\": \"numbered\",\n \"source_line\": 97,\n \"title\": \"Ошибка анализа: не надо слепо включать smtpSsl=true при smtpPort=587.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"17417cdf64e208886d887fd449a612090a81aa5a8f47944f17993821ad5e8235\"\n },\n {\n \"id\": \"ERR-N-26-L102\",\n \"kind\": \"numbered\",\n \"source_line\": 102,\n \"title\": \"Ошибка: openssl -crlf вместе с ручным CRLF ломает SMTP AUTH checker.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"3f83176fadcfeb2318a3e3fc75c13e3944d3fef3c0aa7aef40db3736b27e794a\"\n },\n {\n \"id\": \"ERR-N-27-L108\",\n \"kind\": \"numbered\",\n \"source_line\": 108,\n \"title\": \"Ошибка: Python heredoc внутри ssh сломал not_ok диагностику.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"437f761644210f81c8ca3333410a58585c7b74c54a983b9d84159d98c49b4847\"\n },\n {\n \"id\": \"ERR-N-28-L112\",\n \"kind\": \"numbered\",\n \"source_line\": 112,\n \"title\": \"Ошибка: docker compose ls --format json | wc -l дал ложный COMPOSE_PROJECT_COUNT=1.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"752e0913f85275527a448868cce24aa85cf06949ea82e066e4dc0fc2e857d1ad\"\n },\n {\n \"id\": \"ERR-N-29-L116\",\n \"kind\": \"numbered\",\n \"source_line\": 116,\n \"title\": \"Ошибка: monitoring compact status искал неверные имена health-файлов.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"61df0580bfb53db159bb0c3fced23235437e823a877822e0b8aebb7518453d4d\"\n },\n {\n \"id\": \"ERR-N-30-L121\",\n \"kind\": \"numbered\",\n \"source_line\": 121,\n \"title\": \"Ошибка/инцидент: Prometheus API HTTP_CODE=000 после monitoring reference.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"c0dac19354e77153d3f61d53659a8e694c57308bc564e03c833cb68831e88031\"\n },\n {\n \"id\": \"ERR-N-31-L125\",\n \"kind\": \"numbered\",\n \"source_line\": 125,\n \"title\": \"Строгое правило: перед каждой командой сверяться с файлом ошибок и справочником.\",\n \"summary\": \"\",\n \"rule_like\": true,\n \"signature\": \"fa4e49944e2caf8ba4f26a6064bfb465c035e8bc3184244c0314812e81bd6df5\"\n },\n {\n \"id\": \"ERR-N-32-L130\",\n \"kind\": \"numbered\",\n \"source_line\": 130,\n \"title\": \"Ошибка: 153_PROMETHEUS_TARGETS_SETTLED_PROOF содержит SyntaxError из-за вложенного Python внутри SSH.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"13b3234027d04af4194a5352dda2a5bd21ab955751eb161b953637347da8febf\"\n },\n {\n \"id\": \"ERR-N-34-L135\",\n \"kind\": \"numbered\",\n \"source_line\": 135,\n \"title\": \"Ошибка операционного ввода: полный терминальный лог MobaXterm попал в shell.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"9a7930e756982125ac45f47a4adf7dfdb10906bd950db4ddae3659021e01bc64\"\n },\n {\n \"id\": \"ERR-N-33-L140\",\n \"kind\": \"numbered\",\n \"source_line\": 140,\n \"title\": \"Security finding: root authorized_keys на PVE-нодах имел права 777.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"44a7ef2e7d0428388c9b30d12ecfad460caaca457b5d283ebf00ae87c1b2119c\"\n },\n {\n \"id\": \"ERR-N-35-L144\",\n \"kind\": \"numbered\",\n \"source_line\": 144,\n \"title\": \"Ошибка проверки: 160_PVE_ROOT_AUTHORIZED_KEYS_PERMISSIONS_FIXED не доказал исправление прав.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"336bfad03f4600ddc0d8996bdbbebeae51cfa0c1e2281ae1a838b8b4d8bc7dfd\"\n },\n {\n \"id\": \"ERR-N-36-L149\",\n \"kind\": \"numbered\",\n \"source_line\": 149,\n \"title\": \"Quality check: Storage block needs integrity and pve03 capacity coverage review.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"a11f29a5da892017620722200a35bdde4c0fe5aa591de4331bfbee0851e38266\"\n },\n {\n \"id\": \"ERR-N-37-L154\",\n \"kind\": \"numbered\",\n \"source_line\": 154,\n \"title\": \"Coverage gap: pve03_staging missing from disk-space health coverage.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"d2fded36e2e512289b08c31b7fc6872e4efbf61c06e2840e5a2dfef9a8e989f8\"\n },\n {\n \"id\": \"ERR-N-38-L158\",\n \"kind\": \"numbered\",\n \"source_line\": 158,\n \"title\": \"Quality check: Service Dependency Map block needs integrity review.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"79cffb917505e112ac9507770da4068e145c75d2278d50d1034bb594b1da5062\"\n },\n {\n \"id\": \"ERR-N-39-L162\",\n \"kind\": \"numbered\",\n \"source_line\": 162,\n \"title\": \"Ошибка проверки: post deep audit BAD_MARKER scan caught prose false positive.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"6dc6e561ff4c49c39a8d2f7fdac4fbfd9d2b6295212185791fb9091f644a2824\"\n },\n {\n \"id\": \"ERR-N-40-L166\",\n \"kind\": \"numbered\",\n \"source_line\": 166,\n \"title\": \"Ошибка/вывод: VM150 Mail cloud single-file upload failed with 413.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"d7e096eac4cbc2b602601af11ce42bb716d28142a278203aefb65699b5fd4b9c\"\n },\n {\n \"id\": \"ERR-N-41-L170\",\n \"kind\": \"numbered\",\n \"source_line\": 170,\n \"title\": \"Ошибка: post-backup-pass-slo-reconciliation checked pve01 router health on edge-vm.\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"44ceb289e0963e0e97c6e8019e93120ad638f5648e8596b987e7d413eaeb59aa\"\n },\n {\n \"id\": \"ERR-H-L31-ПРАВИЛО-ДЛЯ-СПРАВОЧНИКА\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 31,\n \"title\": \"Правило для справочника\",\n \"summary\": \"Не генерировать большой справочник через интерактивную вставку. Следующий справочник делать только маленькими append-блоками или через файл, созданный вне интерактивного shell. 11. Ошибка: считать offhost OK после failed rsync. Если rsync/scp упал, нельзя проверять latest offhost-файл без сверки имени. Проверка должна подтвердить именно новый архив, например дату 2026_06_30-00_20_59. Старый OFFHOST_ZSTD_OK не закрывает новый backup. 12. Ошибка: широкий secret-поиск по /opt/stacks дал шум. Не искать Cloudflare/SMTP сразу по /opt/stacks, MkDocs site, backups и HTML-отчётам.\",\n \"rule_like\": true,\n \"signature\": \"2c69d96d9f906cc88ab3158941ba041ca57ddee1e03554fad588eea96041930a\"\n },\n {\n \"id\": \"ERR-H-L174-ASSISTANT-COMMAND-BATCHING-RULE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 174,\n \"title\": \"ASSISTANT_COMMAND_BATCHING_RULE_20260630\",\n \"summary\": \"- Rule: combine related infrastructure checks, proof generation, reference updates and validation scans into one safe batch command when practical. - Do not split work into many small commands when the operations are low-risk, read-only, or naturally part of one evidence block. - Still keep mandatory preflight checks, short bounded commands, no large paste, no here-docs, no interactive shell traps, and no sensitive values in output. - Split into smaller commands only when safety, rollback, quoting risk, or troubleshooting clarity requires it.\",\n \"rule_like\": true,\n \"signature\": \"73ef09d525c334b9874616d5bafdf17d4898a815d35e736e33df0b83790f6539\"\n },\n {\n \"id\": \"ERR-H-L180-HOME-PORTAL-BASE64-APPLY-FAILURE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 180,\n \"title\": \"HOME_PORTAL_BASE64_APPLY_FAILURE_20260630\",\n \"summary\": \"- Prior home-portal apply attempt 312 failed because a large base64 Python payload decoded as invalid UTF-8. - Rule: avoid large opaque base64 script payloads for homelab changes; use transparent bounded shell/perl/python commands instead.\",\n \"rule_like\": true,\n \"signature\": \"4964d3ff556bb377d7c9d0cb4d487afabc0d8ac69c7fb2a5fccb5ff077a8a478\"\n },\n {\n \"id\": \"ERR-H-L184-HOMELAB-COMMAND-SAFETY-HARDENING-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 184,\n \"title\": \"HOMELAB_COMMAND_SAFETY_HARDENING_20260630\",\n \"summary\": \"- Rule: do not use large opaque payloads, generated base64 scripts, or hidden multi-line script blobs for homelab changes. - Use transparent bounded shell/perl/python commands, or create a temporary readable script and validate it before execution. - Any script used for changes must pass a syntax/parse check before it is allowed to modify files or restart services. - Any Traceback, UnicodeDecodeError, SyntaxError, Permission denied, command-not-found, nonzero critical RC, or failed write marker must force REVIEW or FAILED, never OK. - Success requires both runtime health and content-specific post-checks: expected files changed, expected cards exist, duplicates absent, service/container alive, and no new critical log pattern. - HTTP 200 alone is not a success condition for configuration changes. - If an apply step fails, the next command must explicitly record the failed attempt and then run a corrected transparent apply command.\",\n \"rule_like\": true,\n \"signature\": \"463e648fd9e6d671debe81d12b455563460bc669983e083ef5f0f3b4df2f208b\"\n },\n {\n \"id\": \"ERR-H-L193-HOME-PORTAL-API-ERROR-CLOSURE-RULE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 193,\n \"title\": \"HOME_PORTAL_API_ERROR_CLOSURE_RULE_20260630\",\n \"summary\": \"- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200. - If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed. - External informational widgets such as weather/Open-Meteo must not block the service launcher portal.\",\n \"rule_like\": true,\n \"signature\": \"1a011c6be8607fdacb9571baa017241dcb441a0025f985ecce3d0937a4f7e3b5\"\n },\n {\n \"id\": \"ERR-H-L198-HOME-PORTAL-COMPLEX-QUOTING-FAILURE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 198,\n \"title\": \"HOME_PORTAL_COMPLEX_QUOTING_FAILURE_20260630\",\n \"summary\": \"- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('. - Cause: command was too complex and fragile due to nested shell/perl/python quoting. - Rule: do not use deeply nested one-liners for config edits; first run short state checks, then apply small transparent edits.\",\n \"rule_like\": true,\n \"signature\": \"862820180d4dbc46a845af7c9ec8847aeb21cda3fd7634c0d2afd9b3d14d5d8b\"\n },\n {\n \"id\": \"ERR-H-L203-HOME-PORTAL-ACTIVE-CONFIG-SCOPE-RULE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 203,\n \"title\": \"HOME_PORTAL_ACTIVE_CONFIG_SCOPE_RULE_20260630\",\n \"summary\": \"- Rule: Homepage cleanup validation must check active config files only, not logs or backup files. - Active files are services.yaml, bookmarks.yaml, widgets.yaml, settings.yaml and docker.yaml under /opt/stacks/homepage/config. - Matches in /logs/ or *.bak* are historical evidence only and must not make cleanup REVIEW unless the active config still contains the target.\",\n \"rule_like\": true,\n \"signature\": \"6bfd4c200f43d77ecd18a1c61c377b4c0588a028fb0ca6d565b2361d16a74967\"\n },\n {\n \"id\": \"ERR-H-L208-HOME-PORTAL-LINK-VALIDATION-NPMPLUS-DEFAULT-RULE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 208,\n \"title\": \"HOME_PORTAL_LINK_VALIDATION_NPMPLUS_DEFAULT_RULE_20260630\",\n \"summary\": \"- Rule: Homepage link validation must not treat HTTP 200 as success if the body is the NPMPlus default “Congratulations” / host-not-set-up page. - Green siteMonitor dots can be false positives when NPMPlus returns its default vhost page. - A portal card is openable only when DNS, reverse-proxy host mapping, and target content are all valid.\",\n \"rule_like\": true,\n \"signature\": \"90363c457faef01bbe65d3e246073788fb426212371de346152c7776d246413b\"\n },\n {\n \"id\": \"ERR-H-L213-ROUTER-CLI-STDIN-APPLY-FAILURE-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 213,\n \"title\": \"ROUTER_CLI_STDIN_APPLY_FAILURE_20260630\",\n \"summary\": \"- Context: attempted Netcraze router ACL apply through SSH stdin/multiline for Homepage Moscow Router monitor fix. - Evidence: 391_MOSCOW_ROUTER_EDGE_VM_WEBADMIN_ACL_APPLY_PROOF.txt. - Failure: STDIN_SHOW_OK=no, APPLY_SKIPPED=stdin_show_failed, CONFIG_HAS_EDGE_5080=no, CONFIG_HAS_EDGE_5083=no. - Rule: do not apply Netcraze config through stdin/multiline SSH; prove CLI input mode first.\",\n \"rule_like\": true,\n \"signature\": \"7a151b61221247f7e5ecc6488283190d82b489caa9309675c08baded25d7fc6f\"\n },\n {\n \"id\": \"ERR-H-L219-ROUTER-CLI-PROBE-STDIN-CONSUMPTION-ERROR-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 219,\n \"title\": \"ROUTER_CLI_PROBE_STDIN_CONSUMPTION_ERROR_20260630\",\n \"summary\": \"- Context: ACL syntax read-only probe loop executed only one command because ssh consumed the loop stdin. - Evidence: 393_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_PROOF.txt showed COMMAND_COUNT=1. - Fix pattern: use ssh -n or redirect SSH stdin away from the command-list loop for all future SSH-in-loop probes.\",\n \"rule_like\": false,\n \"signature\": \"c801838e0f61bbd4f111bc30b21d530e69124b4448859e0e5f28a93cf0f01fec\"\n },\n {\n \"id\": \"ERR-H-L224-ROUTERBACKUP-READONLY-CLI-PERMISSION-LIMIT-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 224,\n \"title\": \"ROUTERBACKUP_READONLY_CLI_PERMISSION_LIMIT_20260630\",\n \"summary\": \"- Context: routerbackup user can run show running-config but cannot execute configuration commands such as access-list. - Evidence: 394_ROUTER_CLI_ACL_SYNTAX_READONLY_PROBE_FIXED_PROOF.txt. - Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli]. - Rule: do not attempt ACL/config changes as routerbackup; use admin/operator router access or router Web UI for write changes.\",\n \"rule_like\": true,\n \"signature\": \"1d8075c128dbd4b8f34929055e7c4b7c8cfda8f44f0b7874f04afbcc745d5d82\"\n },\n {\n \"id\": \"ERR-H-L230-PROOF-SUMMARY-EXTRACTION-BLANK-20260630\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 230,\n \"title\": \"PROOF_SUMMARY_EXTRACTION_BLANK_20260630\",\n \"summary\": \"- Context: proof 395 tried to read COMMAND_COUNT/ZERO_RC_COUNT from proof files 393/394, but those summary fields were printed to console and were not persisted inside the proof files. - Evidence: 395_ROUTER_CLI_ERROR_REGISTER_UPDATE_PROOF.txt has blank P393_COMMAND_COUNT/P394_COMMAND_COUNT/P394_ZERO_RC_COUNT fields. - Rule: when creating follow-up proof summaries, derive counts from persisted proof body lines such as CMD_INDEX and CMD_RC, or write summary fields into the proof file itself.\",\n \"rule_like\": false,\n \"signature\": \"1fd6a207e921a706e6d8d3e80984d0436e67192bb56d03c91d20f6b33eba15cd\"\n },\n {\n \"id\": \"ERR-H-L235-ROUTER-ACL-UI-RULE-DELETE-AND-RESTORE-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 235,\n \"title\": \"ROUTER_ACL_UI_RULE_DELETE_AND_RESTORE_20260701\",\n \"summary\": \"- Context: while moving Moscow Router monitor rules to _WEBADMIN_Bridge1, two _WEBADMIN_Bridge0 Nextcloud Talk TURN rules were accidentally removed. - Deleted/restored rules: Home [PRIVATE_IP]/24 to Nextcloud [PRIVATE_IP] TCP/UDP 3478. - Verification proof: 399_ROUTER_ACL_RESTORE_AFTER_UI_PROOF.txt. - Rule: after manual router Web UI ACL edits, compare _WEBADMIN_Bridge0, _WEBADMIN_Bridge1 and _WEBADMIN_GigabitEthernet1 before continuing.\",\n \"rule_like\": false,\n \"signature\": \"973a47bd8e8f388786157b6d50b53de9fc96b31e582a9645f3cf4189b74ac4fe\"\n },\n {\n \"id\": \"ERR-H-L241-ROUTER-MONITOR-NODE-LOOKUP-PROBE-BUG-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 241,\n \"title\": \"ROUTER_MONITOR_NODE_LOOKUP_PROBE_BUG_20260701\",\n \"summary\": \"- Context: proof 400 tested Moscow router HTTPS monitor from Homepage container with custom Node lookup callback. - Failure: NODE_HTTPS_RESOLVE_ERROR=ERR_INVALID_IP_ADDRESS and NODE_HTTPS_RESOLVE_INSECURE_ERROR=ERR_INVALID_IP_ADDRESS. - Impact: Node HTTPS resolve subtest is invalid; curl --resolve results remain valid. - Rule: do not rely on custom Node lookup callback until separately tested; prefer curl --resolve or simple direct Node HTTP checks.\",\n \"rule_like\": true,\n \"signature\": \"97151454d5ffda69d45b802ed15dc82057b02632e141b3729fa8ad47a959c556\"\n },\n {\n \"id\": \"ERR-H-L247-ROUTER-HTTP-SERVICE-FORBIDDEN-FROM-EDGE-VM-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 247,\n \"title\": \"ROUTER_HTTP_SERVICE_FORBIDDEN_FROM_EDGE_VM_20260701\",\n \"summary\": \"- Context: Moscow Router Homepage monitor after ACL fix. - Evidence: proofs 399, 400, 401. - Observed: edge-vm TCP to [PRIVATE_IP]:5080/5083 is OK, but router HTTP/HTTPS returns 403 for all tested paths. - Impact: Homepage siteMonitor cannot be made green via direct router HTTP URL until router web service allows the Bridge1/Proxmox/protected segment or an explicit safe monitor endpoint is used. - Rule: do not keep changing ACL for this symptom; ACL is already correct. Fix service access scope or use a dedicated monitor endpoint.\",\n \"rule_like\": true,\n \"signature\": \"b0de3053a46b06841cbaf6ea5945be49ad8d451c7c8453d899d01a9f520cc384\"\n },\n {\n \"id\": \"ERR-H-L254-PY-COMPILE-PYC-PERMISSION-ERROR-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 254,\n \"title\": \"PY_COMPILE_PYC_PERMISSION_ERROR_20260701\",\n \"summary\": \"- Context: installing edge-vm Moscow router health endpoint. - Mistake: assistant used `python3 -m py_compile` on a script installed under /usr/local/sbin as root. - Actual impact: non-root syntax check attempted to create /usr/local/sbin/__pycache__ and failed with Permission denied. - Recovery: validate syntax with `ast.parse` instead of py_compile, because it does not write .pyc files. - Rule: do not use py_compile against root-owned system paths from an unprivileged user.\",\n \"rule_like\": true,\n \"signature\": \"683a661513a8d46a89bf6492c8f12564f6f9fe262ce16b0f6487f63c0f2666f0\"\n },\n {\n \"id\": \"ERR-H-L261-ROUTER-HOME-IP-DOCKER-ROUTE-CONFLICT-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 261,\n \"title\": \"ROUTER_HOME_IP_DOCKER_ROUTE_CONFLICT_20260701\",\n \"summary\": \"- Context: Moscow Router Homepage siteMonitor attempt using http://[PRIVATE_IP]:5080. - Evidence: proof 405 showed route to [PRIVATE_IP] via Docker bridge and Homepage container ECONNRESET. - Impact: do not use [PRIVATE_IP] directly as Homepage siteMonitor from edge-vm. - Mitigation: use dedicated edge-vm health endpoint that checks router TCP [PRIVATE_IP]:5080 and returns HTTP 200/503 for Homepage.\",\n \"rule_like\": true,\n \"signature\": \"5daeb3cf0b7d3bf64ddf6ca87e958f73d0715672617706dee233fd739fb08c2c\"\n },\n {\n \"id\": \"ERR-H-L267-HOMEPAGE-ROUTER-MOSCOW-YAML-TITLE-SHAPE-MISMATCH-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 267,\n \"title\": \"HOMEPAGE_ROUTER_MOSCOW_YAML_TITLE_SHAPE_MISMATCH_20260701\",\n \"summary\": \"- Context: applying Moscow Router Homepage siteMonitor health endpoint. - Mistake: assistant apply script matched only property-style cards with `title`, but active Homepage YAML can use service-name-key style like `- Service Name:`. - Evidence: proof 407 returned YAML_CHANGE_COUNT=0 and APPLY_SKIPPED=unexpected_match_count while health endpoint proof 406 was OK. - Recovery: use YAML-aware updater that handles both property-style and service-name-key style cards.\",\n \"rule_like\": false,\n \"signature\": \"d7ed6a0ecfb19584319e97626e76661541c675e9125e4bf4ff0a407c9718cdd0\"\n },\n {\n \"id\": \"ERR-H-L273-HOMEPAGE-ROUTER-MOSCOW-APPLY-SCRIPT-SYNTAX-ERROR-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 273,\n \"title\": \"HOMEPAGE_ROUTER_MOSCOW_APPLY_SCRIPT_SYNTAX_ERROR_20260701\",\n \"summary\": \"- Context: applying Moscow Router Homepage health endpoint. - Mistake: assistant generated Python script with invalid f-string escaping. - Evidence: proof 408 showed SyntaxError in /tmp/homepage-router-moscow-apply-fixed.py. - Actual impact: YAML was not changed, so Homepage green dot could not appear. - Recovery: stop applying blind updater; first discover exact active YAML shape, then apply minimal YAML-aware edit. - Rule: validate generated Python syntax locally before scp/remote execution; avoid f-strings in generated maintenance scripts.\",\n \"rule_like\": true,\n \"signature\": \"057b7e036316f6f240cac06b9f46e98d8399df689f82b903cc9ef630afe09f94\"\n },\n {\n \"id\": \"ERR-H-L281-FORUM-PROD-VM160-FIRST-BOOT-SSH255-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 281,\n \"title\": \"FORUM_PROD_VM160_FIRST_BOOT_SSH255_20260701\",\n \"summary\": \"- Context: clean rebuilt VM160 first boot. - Symptom: PING_OK=yes and TCP22_OK=yes, but SSH_RC=255. - Rule: do not proceed with OS baseline until SSH failure is diagnosed; likely old known_hosts key or cloud-init/root-key issue.\",\n \"rule_like\": true,\n \"signature\": \"5e0359dc6b12e39f533e9060876f5c5f830b43cf2dbbe0441d6876fee5f410a3\"\n },\n {\n \"id\": \"ERR-H-L286-FORUM-PROD-VM160-NESTED-HOSTNAME-PROOF-QUOTING-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 286,\n \"title\": \"FORUM_PROD_VM160_NESTED_HOSTNAME_PROOF_QUOTING_20260701\",\n \"summary\": \"- Context: VM160 first SSH proof after rebuild. - Issue: command substitution $(hostname) inside nested ssh was expanded on pve02 before entering VM160. - Impact: VM_HOSTNAME=pve02 in proof 426 is not valid VM identity evidence. - Rule: for nested SSH identity checks, run literal hostname commands without local command substitution.\",\n \"rule_like\": false,\n \"signature\": \"c7174e4703945abd82004fea3fa9da0ff3efa1f2a89e6d2315cacf9c27b6d761\"\n },\n {\n \"id\": \"ERR-H-L292-FORUM-PROD-VM160-SWAP-PROOF-QUOTING-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 292,\n \"title\": \"FORUM_PROD_VM160_SWAP_PROOF_QUOTING_20260701\",\n \"summary\": \"- Context: VM160 swapfile proof 429. - Issue: nested command substitution expanded on pve02, producing invalid SWAPFILE/FSTAB proof lines. - Impact: proof 429 is not valid closure evidence even though swap was active. - Rule: avoid nested $(...) in VM SSH proofs; use literal remote commands and clean proof.\",\n \"rule_like\": true,\n \"signature\": \"3dc2ea2b0a54adb8886979de4e01cb5f23704446f5b2198af9bdacc951359e4c\"\n },\n {\n \"id\": \"ERR-H-L298-FORUM-UPLOAD-ARCHIVES-VM-VERIFY-QUOTING-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 298,\n \"title\": \"FORUM_UPLOAD_ARCHIVES_VM_VERIFY_QUOTING_20260701\",\n \"summary\": \"- Context: proof 446 copy/check archives inside VM160. - Issue: nested SSH quoting expanded shell variables incorrectly, producing gzip checks against empty .gz and blank TAR_TOP lines. - Impact: SCP_TO_VM_RC=0 and file size list are valid, but GZIP/TAR proof lines in 446 are invalid. - Rule: verify VM archive integrity locally inside forum-prod, not through nested quoted loops.\",\n \"rule_like\": false,\n \"signature\": \"0e4a3a2f7c179f46c235e1ffa176b57485c88b4f7e7314bb89b6acb6683fe448\"\n },\n {\n \"id\": \"ERR-H-L304-FORUM-PROD-BULK-IMPORT-PHP85-EMPTY-FRONTEND-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 304,\n \"title\": \"FORUM_PROD_BULK_IMPORT_PHP85_EMPTY_FRONTEND_20260701\",\n \"summary\": \"- Context: rebuilt VM160, five XenForo forums imported in bulk on PHP 8.5.7. - Symptom: all five forums returned HTTP 200 with empty body; XenForo logged Template public:PAGE_CONTAINER/forum_list is unknown. - Additional failed approach: proof 480 ran xf-dev:rebuild-caches on CodeVipe and returned RC=1, not a valid fix. - Decision: stop bulk repair, treat current VM160 as lab, find a working CodeVipe recipe first, then destroy/rebuild cleanly for all five forums. - Rule: no more mass rebuild/repair commands across all forums until single-forum CodeVipe recipe is proven.\",\n \"rule_like\": false,\n \"signature\": \"ea3c74648eece7c2a94559d8f7385ea4a126b399e3a161c5113e69a7b8d05f93\"\n },\n {\n \"id\": \"ERR-H-L311-XENFORO-REQUIREMENTS-DIRECT-DOWNLOAD-403-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 311,\n \"title\": \"XENFORO_REQUIREMENTS_DIRECT_DOWNLOAD_403_20260701\",\n \"summary\": \"- Context: Tried to download XenForo requirements ZIP directly from xenforo.com inside forum-prod. - Issue: curl returned HTTP 403; unzip then failed because the ZIP was not downloaded. - Evidence: proof 491 showed DOWNLOAD_RC=22, ZIP_LIST_RC=9, UNZIP_RC=9. - Impact: proof 491 is not a valid server compatibility test. - Rule: use operator-uploaded xenforo23-requirements-test.zip from /root/forum-upload instead of relying on direct curl download.\",\n \"rule_like\": false,\n \"signature\": \"13164e6750f679553f0bb7cb13f4b895f345dc3f76c978349dfd30b93294206b\"\n },\n {\n \"id\": \"ERR-H-L318-PVE01-UNZIP-MISSING-FOR-XF-REQUIREMENTS-VERIFY-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 318,\n \"title\": \"PVE01_UNZIP_MISSING_FOR_XF_REQUIREMENTS_VERIFY_20260701\",\n \"summary\": \"- Context: proof 492 tried to inspect uploaded xenforo23-requirements-test.zip on pve01. - Issue: pve01 does not have unzip installed, so ZIP_TEST_RC=127 was not a ZIP integrity result. - Impact: proof 492 confirmed file presence only, not archive validity. - Rule: verify ZIP with Python zipfile or inside forum-prod rather than installing unzip on Proxmox just for inspection.\",\n \"rule_like\": false,\n \"signature\": \"341e814b6c0aab80d0bde631353118c8c04593d918a2ed7b60551d5bc3043148\"\n },\n {\n \"id\": \"ERR-H-L324-FRESH5-DEPLOY-SUCCESS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 324,\n \"title\": \"FRESH5_DEPLOY_SUCCESS_20260701\",\n \"summary\": \"- Context: VM160 was destroyed/recreated, then five XenForo forums were deployed from fresh ZIP backups. - Result: proof 513 confirms all five forums locally healthy. - Note: proof 511 initially failed dsmods path detection; proof 512 fixed dsmods by locating public_html via src/XF.php marker. - Rule: future restores should preserve internal_data/code_cache or rebuild compiled template cache before smoke testing.\",\n \"rule_like\": false,\n \"signature\": \"f46e9779cb65eebc42015f7042a4f55f0a8aa1e59e647eb29d728e48ed6069b3\"\n },\n {\n \"id\": \"ERR-H-L330-NPMPLUS-SQLITE-PASTE-FAILURE-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 330,\n \"title\": \"NPMPLUS_SQLITE_PASTE_FAILURE_20260701\",\n \"summary\": \"- Context: NPMplus SQLite inspection command was too complex and a Python fragment was pasted into bash. - Issue: shell entered multiline prompt and produced syntax errors. - Impact: do not trust that SQLite inspection attempt. - Rule: avoid nested SSH heredoc/Python for this task; use simpler commands, generated nginx configs, or uploaded/local script files.\",\n \"rule_like\": true,\n \"signature\": \"d1f2dda1b36bc12803264780b71545d7d9b00c4bdb88ef2fa85405e9d27cdf6f\"\n },\n {\n \"id\": \"ERR-H-L336-NPMPLUS-PUBLISH-SCRIPT-WRONG-ENV-SOURCE-525-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 336,\n \"title\": \"NPMPLUS_PUBLISH_SCRIPT_WRONG_ENV_SOURCE_525_20260701\",\n \"summary\": \"- Context: proof 525 tried to create forum proxy hosts in NPMplus. - Issue: script sourced /opt/npmplus/.env, but INITIAL_ADMIN_EMAIL/PASSWORD were not present there; they are in the npmplus container env. - Impact: no forum proxy hosts were created by proof 525. - Rule: read NPMplus API login values from docker inspect env internally, never print them.\",\n \"rule_like\": true,\n \"signature\": \"58aa4eeee2841a42e270226a408177af9a099631e2a47857c825707b2ac523ff\"\n },\n {\n \"id\": \"ERR-H-L342-NPMPLUS-API-LOGIN-MISSED-HTTPS81-AND-JSON-ESCAPE-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 342,\n \"title\": \"NPMPLUS_API_LOGIN_MISSED_HTTPS81_AND_JSON_ESCAPE_20260701\",\n \"summary\": \"- Context: NPMplus API login attempts in proofs 526/527 failed. - Issue: commands missed the likely HTTPS admin endpoint on port 81 and built JSON by string interpolation instead of jq escaping. - Impact: no proxy hosts were created by 526/527. - Rule: for NPMplus API use jq-generated JSON and test https://127.0.0.1:81/api/tokens with -k before publish.\",\n \"rule_like\": false,\n \"signature\": \"141ce6d2777003569c531b5308e56cffaea92dc113fdb0c9e9ebe45815805101\"\n },\n {\n \"id\": \"ERR-H-L348-NPMPLUS-API-AUTH-UNAVAILABLE-MANUAL-ROUTES-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 348,\n \"title\": \"NPMPLUS_API_AUTH_UNAVAILABLE_MANUAL_ROUTES_20260701\",\n \"summary\": \"- Context: NPMplus API login attempts failed even though the forum VM and edge-to-forum connectivity are healthy. - Issue: API credentials from container initial env are not accepted by current NPMplus API. - Impact: do not use NPMplus API for this publish path. - Rule: publish forum routes through backed-up manual nginx proxy_host configs, then issue proper certificates after DNS points to edge.\",\n \"rule_like\": true,\n \"signature\": \"d5a4f4c6312277e9e2c1487b38c4bf554f4b9618fad9e2c5328fe8324e26a555\"\n },\n {\n \"id\": \"ERR-H-L354-EDGE-CERTBOT-TOKEN-FILE-PERMISSION-535-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 354,\n \"title\": \"EDGE_CERTBOT_TOKEN_FILE_PERMISSION_535_20260701\",\n \"summary\": \"- Context: proof 535 attempted DNS-01 certificate issue for five forum domains. - Issue: /tmp/forum_cf_token.env was root-owned mode 600, so debian user could not source it directly. - Impact: certbot ran without token and all certificate attempts failed; DNS cutover must not proceed from proof 535. - Rule: read Cloudflare token via sudo without printing it, then create temporary certbot credentials file and remove it after use.\",\n \"rule_like\": true,\n \"signature\": \"4566b4ef97181c31efcb32eefd3ed52633883fc929e98d4607a6c48671434582\"\n },\n {\n \"id\": \"ERR-H-L360-EDGE-CERTBOT-FAILED-DNS-CUTOVER-ABORTED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 360,\n \"title\": \"EDGE_CERTBOT_FAILED_DNS_CUTOVER_ABORTED_20260701\",\n \"summary\": \"- Context: edge manual routes are working locally, but certbot DNS-01 certificate issue failed in proof 538. - Issue: all five certbot attempts returned RC=1; proof 539 correctly aborted DNS cutover because edge cert precheck was not ready. - Impact: Cloudflare DNS records were not changed by proof 539; public DNS still points to old IP 87.236.18.45. - Rule: inspect certbot logs and Cloudflare token zone permissions before retrying certificate issuance or DNS cutover.\",\n \"rule_like\": false,\n \"signature\": \"7fc8883c0f715d038cc77e38a9088d6cb18290d2456232e9e0a13b1ba094d256\"\n },\n {\n \"id\": \"ERR-H-L366-CLOUDFLARE-TOKEN-NO-ZONE-ACCESS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 366,\n \"title\": \"CLOUDFLARE_TOKEN_NO_ZONE_ACCESS_20260701\",\n \"summary\": \"- Context: Cloudflare token discovered on edge verified as a token but could not access forum zones. - Evidence: proof 542 showed all five zones missing and DNS record create probes failed. - Impact: certbot DNS-01 failed and DNS cutover was correctly aborted; public DNS still points to old hosting. - Rule: install a new Cloudflare token with Zone Read and DNS Edit for the five forum zones before retrying certs/DNS.\",\n \"rule_like\": false,\n \"signature\": \"3ba4fdbb1dcc5d0beeb215153e4471c19e29f4080ff3d061e77154f44de9a898\"\n },\n {\n \"id\": \"ERR-H-L372-FORUM-PUBLICATION-FINAL-SUCCESS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 372,\n \"title\": \"FORUM_PUBLICATION_FINAL_SUCCESS_20260701\",\n \"summary\": \"- Context: five XenForo forums were rebuilt on VM160, routed through edge NPMplus, issued Let’s Encrypt certs using a corrected Cloudflare token, and cut over in DNS. - Result: final public proof 546 passed. - Important corrections: initial NPMplus API route failed; manual Nginx routes were used. Initial Cloudflare token had no zone access; corrected token passed zone/DNS probe.\",\n \"rule_like\": false,\n \"signature\": \"bec7980ce2230b8ecf69cfc694cc56bbd3dedc45da55ed2df567e8f41b228638\"\n },\n {\n \"id\": \"ERR-H-L377-FINAL-SNAPSHOT-NAME-TOO-LONG-556-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 377,\n \"title\": \"FINAL_SNAPSHOT_NAME_TOO_LONG_556_20260701\",\n \"summary\": \"- Context: proof 556 final health gate passed for all five public forums. - Issue: Proxmox snapshot failed because generated snapshot name exceeded 40 characters. - Evidence: qm snapshot returned snapname value may only be 40 characters long. - Impact: forum health was OK, but proof 556 snapshot step was not completed. - Fix: rerun snapshot with short name.\",\n \"rule_like\": false,\n \"signature\": \"c3d65c37278c1e067af0f6dc1d7eb36fa6e75b03113162e66adf06c483bedc76\"\n },\n {\n \"id\": \"ERR-H-L384-CLOUDFLARE-DNS-AUDIT-TXT-SUBSTRING-GAP-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 384,\n \"title\": \"CLOUDFLARE_DNS_AUDIT_TXT_SUBSTRING_GAP_20260701\",\n \"summary\": \"- Context: prior Cloudflare DNS audit checked web A/AAAA/CNAME state for root/www records. - Issue: it did not deeply inspect TXT/SPF contents for old provider IP substrings. - Evidence: user screenshot showed SPF TXT containing old 87.236.18.* value. - Impact: web routing is OK, but mail-related DNS may still contain stale provider data. - Rule: perform full DNS record audit including TXT/SPF/DKIM/DMARC/MX/CDN and substring search for old IPs before DNS cleanup.\",\n \"rule_like\": false,\n \"signature\": \"260c1f016ae9bf9efb5888bb54809e742681b512223781efac0a4cc8d8cdf2c7\"\n },\n {\n \"id\": \"ERR-H-L391-CLOUDFLARE-DNS-CLEANUP-DRYRUN-QUOTE-563-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 391,\n \"title\": \"CLOUDFLARE_DNS_CLEANUP_DRYRUN_QUOTE_563_20260701\",\n \"summary\": \"- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt. - Impact: proof 563 is invalid and no DNS cleanup was performed by it. - Rule: use copied scripts for Cloudflare DNS mutation instead of fragile inline nested quotes.\",\n \"rule_like\": false,\n \"signature\": \"3837def1e15ccdc91af766587f44bd7e98885fb25b88ca3d62e65d13698f62d1\"\n },\n {\n \"id\": \"ERR-H-L396-SPF-DUPLICATE-AFTER-565-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 396,\n \"title\": \"SPF_DUPLICATE_AFTER_565_20260701\",\n \"summary\": \"- Context: SPF cleanup command 565 attempted to replace stale SPF records. - Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained. - Impact: domains temporarily had duplicate SPF records, which is invalid for mail validation. - Fix: delete all SPF TXT records for the five forum zones, then create exactly one v=spf1 -all TXT per zone.\",\n \"rule_like\": false,\n \"signature\": \"4a4b1cba0ac945aef40797821d7a9a774728f30fb5e56c34599575e1b5b40ba8\"\n },\n {\n \"id\": \"ERR-H-L402-MSMTP-SECRET-SOURCE-PARSE-ERROR-576-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 402,\n \"title\": \"MSMTP_SECRET_SOURCE_PARSE_ERROR_576_20260701\",\n \"summary\": \"- Context: proof 576 installed msmtp but sendmail auth test failed. - Issue: [SENSITIVE_PATH] was sourced as shell, but SMTP password contained shell-special characters; source failed and auth used an invalid/truncated secret path. - Impact: msmtp package installed, but mail sending was not proven working. - Rule: never source SMTP secret files containing arbitrary passwords; store password base64 and use msmtp passwordeval helper accessible to www-data.\",\n \"rule_like\": true,\n \"signature\": \"6288f1fcabcdeda1884be858bea052ff9dbb35a65054f5cff6315a350dd7ef12\"\n },\n {\n \"id\": \"ERR-H-L408-FORUM-MSMTP-MAIL-TRANSPORT-STILL-FAILING-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 408,\n \"title\": \"FORUM_MSMTP_MAIL_TRANSPORT_STILL_FAILING_20260701\",\n \"summary\": \"- Context: attempted to fix msmtp config with passwordeval helper. - Issue: one or more sendmail/PHP mail tests still failed. - Impact: XenForo mail sending is not yet proven. - Next step: verify SMTP credential/password and provider policy for noreply@pvepro.ru.\",\n \"rule_like\": false,\n \"signature\": \"31d614a3bb69ec05a33a5be14388347a438e13f1e28a7ec58207273159e00eae\"\n },\n {\n \"id\": \"ERR-H-L414-SMTP-PASSWORD-EXPOSED-ROTATE-REQUIRED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 414,\n \"title\": \"SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701\",\n \"summary\": \"- Context: SMTP password was exposed in terminal output during failed msmtp setup. - Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line. - Impact: treat that SMTP password as compromised. - Required action: rotate the mailbox/app password in the mail provider panel before any further SMTP testing. - Rule: never store arbitrary passwords as shell-sourceable KEY=value files; use non-printing secret storage only.\",\n \"rule_like\": true,\n \"signature\": \"88ae0aeb13d948e6f2cb2d83dc2a9464e4de1805cdb4271cf78f3d315f7095c9\"\n },\n {\n \"id\": \"ERR-H-L421-NETBIRD-MAILCOW-REPORTED-DOWN-AFTER-SMTP-INCIDENT-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 421,\n \"title\": \"NETBIRD_MAILCOW_REPORTED_DOWN_AFTER_SMTP_INCIDENT_20260701\",\n \"summary\": \"- Context: user reported NetBird and Mailcow became unavailable after forum mail-transport work. - Impact: treat as active incident until service reachability and container/VM state are proven. - Rule: do not change DNS, firewall, proxy, mail, NetBird, or Mailcow configs before read-only triage proof.\",\n \"rule_like\": true,\n \"signature\": \"b0eef26c0503e63d8a8c5a294ea1e124ab5e6ac8936d2ccaf23cd9b74fc5ebf3\"\n },\n {\n \"id\": \"ERR-H-L426-FORUM-SMTP-FAILED-CONFIG-DISABLED-AFTER-MAILCOW-BAN-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 426,\n \"title\": \"FORUM_SMTP_FAILED_CONFIG_DISABLED_AFTER_MAILCOW_BAN_20260701\",\n \"summary\": \"- Context: forum-prod SMTP/msmtp test used invalid or compromised credentials and triggered Mailcow netfilter warnings/ban for 95.84.154.183. - Impact: forum mail transport is not proven and must remain disabled until SMTP password is rotated and tested once carefully. - Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth. - Rule: do not test SMTP auth again until a fresh rotated mailbox/app password is available.\",\n \"rule_like\": true,\n \"signature\": \"70dc70d8c15e3f30611c45123bf2c3b27c86bf66d703df3e4232057d380801d6\"\n },\n {\n \"id\": \"ERR-H-L432-FORUM-OLD-CODEVIPE-CLOUD-BACKUP-STALE-AFTER-FRESH5-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 432,\n \"title\": \"FORUM_OLD_CODEVIPE_CLOUD_BACKUP_STALE_AFTER_FRESH5_20260701\",\n \"summary\": \"- Context: pve02 codevipe-cloud-backup.timer existed before fresh five-forum rebuild. - Issue: old script points to /root/backups/codevipe and old remote path app/codevipe/forum-prod/local. - Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory contains old codevipe-public/codevipe.sql files, not current five-forum archive. - Impact: old timer must not be treated as valid current backup for all five forums. - Fix: create a new forum-prod fresh5 cloud backup using existing pve02 Mail-cloud rclone crypt remotes.\",\n \"rule_like\": true,\n \"signature\": \"3baaa9a93ec07abd251041838b9602a52b143ef8e07a03dde6928db27eb2b44f\"\n },\n {\n \"id\": \"ERR-H-L439-FORUM-RESTORE-DRILL-610-STATUS-FLAG-BUG-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 439,\n \"title\": \"FORUM_RESTORE_DRILL_610_STATUS_FLAG_BUG_20260701\",\n \"summary\": \"- Context: restore drill proof 610 downloaded Mail.ru Cloud backup, verified split part SHA256, reconstructed archive, matched full archive SHA256, and confirmed all five forum DB/file archives in tar. - Issue: final RESTORE_DRILL_OK was set to no because the script tested an unassigned shell variable RCLONE_REMOTE_PVE02_MAIL_01_RC. - Impact: proof 610 final status flag is invalid, but its detailed integrity checks are valid. - Fix: rerun corrected restore validation against the downloaded cloud backup artifacts and produce proof 612.\",\n \"rule_like\": false,\n \"signature\": \"fd5f6f353d76ced7d2deb5e924a183c516c80e7329d8b9d5c4be0627b5b92d41\"\n },\n {\n \"id\": \"ERR-H-L445-FORUM-XENFORO-MAIL-SMOKE-TEST-FAILED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 445,\n \"title\": \"FORUM_XENFORO_MAIL_SMOKE_TEST_FAILED_20260701\",\n \"summary\": \"- XenForo-level mail smoke test did not return success for all five forums. - Check proof 623 and msmtp log before retrying.\",\n \"rule_like\": false,\n \"signature\": \"c48950d0c079bf801e38017835444fdd04a5354397908bbd864f625e7a917b3d\"\n },\n {\n \"id\": \"ERR-H-L449-XENFORO-MAIL-SMOKE-623-INVALID-ROOT-PATH-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 449,\n \"title\": \"XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701\",\n \"summary\": \"- Context: custom XenForo mail smoke test 623 failed with Could not open input file /root/scripts/623_xenforo_mail_smoke.php. - Cause: the script was run as www-data from /root/scripts; /root is not traversable by www-data. - Impact: proof 623 is invalid and should not be used to judge mail delivery. - Superseding evidence: user ran XenForo built-in outgoing email test; message was delivered to Yandex but placed in Spam. - Next step: investigate deliverability/authentication headers and pvepro.ru SPF/DKIM/DMARC/PTR.\",\n \"rule_like\": false,\n \"signature\": \"ad9141334766e53e58aef9b0fb014ebfbfe556d3d36615b5f4754e2b29cf36fd\"\n },\n {\n \"id\": \"ERR-H-L456-FORUM-NEXT-CHAT-KNOWN-ERRORS-AND-CLOSED-INCIDENTS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 456,\n \"title\": \"FORUM_NEXT_CHAT_KNOWN_ERRORS_AND_CLOSED_INCIDENTS_20260701\",\n \"summary\": \"\",\n \"rule_like\": false,\n \"signature\": \"d4866c37a32090d3456bbb2b44824481a232211fdef57833b7f7bec57c065690\"\n },\n {\n \"id\": \"ERR-H-L458-CLOSED-CLASSIFIED-INCIDENTS\",\n \"kind\": \"heading\",\n \"level\": 3,\n \"source_line\": 458,\n \"title\": \"Closed / classified incidents\",\n \"summary\": \"- SMTP_PASSWORD_EXPOSED_ROTATE_REQUIRED_20260701: - A previous bad command sourced a raw SMTP secret and printed it. - Treat old password as compromised. - Later persistent SMTP was rebuilt using safe files and verified. - Never print or package secrets. - XENFORO_MAIL_SMOKE_623_INVALID_ROOT_PATH_20260701: - Custom mail proof 623 failed with \\\"Could not open input file\\\". - Cause: PHP was run as www-data from /root/scripts, but /root is not traversable by www-data.\",\n \"rule_like\": true,\n \"signature\": \"eafc804db9ff3f7b3b68c10318b53db82160fe7951c6e68c4301703c2b7abe41\"\n },\n {\n \"id\": \"ERR-H-L491-CURRENT-NON-BLOCKING-ITEMS\",\n \"kind\": \"heading\",\n \"level\": 3,\n \"source_line\": 491,\n \"title\": \"Current non-blocking items\",\n \"summary\": \"- Yandex placed built-in XenForo test mail in Spam despite SPF pass and DKIM pass. - Classification: deliverability/reputation/content filtering, not server failure. - Recommended manual action: click “Не спам!” and add/check pvepro.ru in Yandex Postmaster.\",\n \"rule_like\": false,\n \"signature\": \"bc80a4393b6c0958e5753d3114c2ff3e0554b960a6fed7b7829625f02a9d595a\"\n },\n {\n \"id\": \"ERR-H-L496-SAFETY-RULES-FOR-NEXT-CHAT\",\n \"kind\": \"heading\",\n \"level\": 3,\n \"source_line\": 496,\n \"title\": \"Safety rules for next chat\",\n \"summary\": \"- Do not print secrets. - Do not download or upload: - [SENSITIVE_PATH] - /etc/msmtprc - /etc/msmtp/* - rclone configs - Cloudflare tokens - DB dumps\",\n \"rule_like\": true,\n \"signature\": \"e95262ee083c9c3d1b8587e182598186f1be23e5db9229b5aece7ea336fa6ca5\"\n },\n {\n \"id\": \"ERR-H-L509-PARKED-DOMAINS-STAGE4-DNS01-PREFLIGHT-FAILED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 509,\n \"title\": \"PARKED_DOMAINS_STAGE4_DNS01_PREFLIGHT_FAILED_20260701\",\n \"summary\": \"- Context: parked-domain public apply proof 634. - Issue: DNS-01 based apply was attempted before a clear CAN_APPLY=yes preflight result was visible in terminal output. - Observed: stage4 stopped safely with preflight_failed; CERTBOT_PRESENT=no, CERTBOT_DNS_CLOUDFLARE_PLUGIN=no, and CF_ZONE_ACCESS=no for newfi.ru, hapusya.ru and kingofwolk.ru. - Impact: no parked-domain NPMplus route/certificate changes were applied by stage4; public HTTPS validation still failed certificate verification. - Rule: do not use Cloudflare DNS-01 for these parked domains unless a token with explicit zone access is installed; because DNS already points to edge, prefer a copied-script HTTP-01 flow with temporary ACME challenge routes and content-specific validation.\",\n \"rule_like\": true,\n \"signature\": \"9a52142ed4109dad52b9391a4a479df84e258ff703bfd8904e4ec5947589cd35\"\n },\n {\n \"id\": \"ERR-H-L516-PARKED-DOMAINS-STAGE5-BASH-LOCAL-SETU-BUG-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 516,\n \"title\": \"PARKED_DOMAINS_STAGE5_BASH_LOCAL_SETU_BUG_20260701\",\n \"summary\": \"- Context: parked-domain HTTP-01 apply proof 635. - Issue: edge script used Bash `local id=\\\"$1\\\" ... conf=\\\"$WORK/.../$id.conf\\\"` and `local host=\\\"$1\\\" ... tmp=\\\"$WORK/.../$host.html\\\"` under `set -u`; dependent variables are not safe inside the same local assignment command. - Observed: `id: unbound variable`, `source can not be empty`, then `host: unbound variable`; certbot did not run, final HTTPS routes were not installed. - Impact: local parked page remained OK; public HTTPS remained not closed. - Rule: split dependent Bash local assignments into separate statements, syntax-check copied scripts before execution, and keep content-specific validation for parked routes.\",\n \"rule_like\": false,\n \"signature\": \"c5a97f3f8380803056dcbb2abd47a72f37609e59655bbd63dec7695e4b5e86e0\"\n },\n {\n \"id\": \"ERR-H-L523-PARKED-DOMAINS-STAGE6-TEMP-HTTP-ROOT-VALIDATION-BUG-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 523,\n \"title\": \"PARKED_DOMAINS_STAGE6_TEMP_HTTP_ROOT_VALIDATION_BUG_20260701\",\n \"summary\": \"- Context: parked-domain HTTP-01 fixed apply proof 636. - Issue: temporary HTTP-01 route validation checked HTTP `/` for parked marker, but expected behavior before certificates is root HTTP 301 to HTTPS. - Observed: LOCAL_HTTP_VALIDATE returned HTTP_CODE=301 MARKER_OK=no, script rolled back its temporary route configs and did not run certbot. - Impact: local parked page remained OK; public HTTPS remained not closed. - Rule: validate `/.well-known/acme-challenge/` for HTTP-01 readiness, not HTTP `/`; also scan all active NPMplus nginx config dirs for unmanaged domain conflicts before writing routes.\",\n \"rule_like\": false,\n \"signature\": \"dfb310ca0c4e8c9d03ea17ec7e7c5821f16f7dde5c3b0d217d1ed361f61a8740\"\n },\n {\n \"id\": \"ERR-H-L530-PARKED-DOMAINS-STAGE7-NEWFI-ROOT-ACME-REDIRECT-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 530,\n \"title\": \"PARKED_DOMAINS_STAGE7_NEWFI_ROOT_ACME_REDIRECT_20260701\",\n \"summary\": \"- Context: parked domains HTTP-01 Stage7 stopped before certbot because ACME validation path returned 301 for root newfi.ru while www.newfi.ru, hapusya.ru, www.hapusya.ru, kingofwolk.ru and www.kingofwolk.ru returned 200. - Impact: no final parked-domain HTTPS routes/certificates were applied by Stage7; temporary route configs were rolled back. - Rule: do not run another apply stage until active NPMplus nginx route selection for newfi.ru is diagnosed with nginx -T and Host-header probes.\",\n \"rule_like\": true,\n \"signature\": \"65ab366077a7e1afb168ff94f8b1b7a8b788e1bdca76e6484300ce76c03431f9\"\n },\n {\n \"id\": \"ERR-H-L535-PARKED-DOMAINS-STAGE8-DEFAULT-HTTP-TO-HTTPS-REDIRECT-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 535,\n \"title\": \"PARKED_DOMAINS_STAGE8_DEFAULT_HTTP_TO_HTTPS_REDIRECT_20260701\",\n \"summary\": \"- Context: parked-domain route autopsy proof 638. - Finding: no active NPMplus config contains newfi.ru/hapusya.ru/kingofwolk.ru, but unknown HTTP hosts and ACME paths return 301 to HTTPS from NPMplus default routing. - Impact: HTTP-01 cannot work through the default server. A dedicated temporary ACME server block must be proven before certbot; do not run another certbot apply without a successful ACME-path probe for all six hostnames. - Rule: after every parked-domain temp route attempt, rollback the route and prove rollback before continuing.\",\n \"rule_like\": true,\n \"signature\": \"fb5de90cf615657074d466c74a748ae250def09fa8ad84bedafdc765a6682be2\"\n },\n {\n \"id\": \"ERR-H-L541-PARKED-DOMAINS-STAGE9-NEWFI-EXACT-PROBE-NEEDED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 541,\n \"title\": \"PARKED_DOMAINS_STAGE9_NEWFI_EXACT_PROBE_NEEDED_20260701\",\n \"summary\": \"- Context: parked domains HTTP-01 probe stage9 installed one combined temporary server block for all six hostnames. - Issue: five hostnames returned ACME probe 200, but root newfi.ru still returned 301. - Impact: do not continue certbot/apply until exact newfi.ru route behavior is isolated. - Rule: run a temporary exact newfi.ru probe block and roll it back before any HTTP-01 apply.\",\n \"rule_like\": true,\n \"signature\": \"b6335dafc9b854cc8c775bb66b93a4f927e15e4d1b50bb1b33eed285902c43ac\"\n },\n {\n \"id\": \"ERR-H-L547-PARKED-DOMAINS-STAGE10-EXACT-NEWFI-PROBE-OK-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 547,\n \"title\": \"PARKED_DOMAINS_STAGE10_EXACT_NEWFI_PROBE_OK_20260701\",\n \"summary\": \"- Context: parked-domain Stage10 proof 640. - Result: exact server block for newfi.ru on NPMplus intercepted both root and ACME paths with HTTP 200 and custom X-Parked-Probe markers. - Stage9 combined server_name block passed 5/6 but failed bare newfi.ru, so final HTTP-01 cutover must use separate exact per-host server blocks. - Rule: do not use one combined parked-domain server_name block for these domains; use exact host blocks and content-specific validation.\",\n \"rule_like\": true,\n \"signature\": \"6c28ff0028785a70150cd3be2574ea8ed678ed0e5d8838b93feef353f8d26e7b\"\n },\n {\n \"id\": \"ERR-H-L553-PARKED-DOMAINS-STAGE11-LOW-ID-EXACT-ROUTE-FAILED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 553,\n \"title\": \"PARKED_DOMAINS_STAGE11_LOW_ID_EXACT_ROUTE_FAILED_20260701\",\n \"summary\": \"- Context: parked-domain Stage11 proof 641. - Result: exact per-host route files 700-705 passed nginx -t but bare newfi.ru still hit HTTP->HTTPS redirect on ACME path. - Earlier Stage10 proof showed exact newfi.ru route works when placed at high id 998. - Hypothesis: file/include priority or NPMplus generated ordering makes low id 700 unsuitable for bare newfi.ru in this runtime. - Rule: do not retry parked-domain HTTP-01 with low id 700-705. First prove fixed high-id exact blocks, then apply with those exact ids.\",\n \"rule_like\": true,\n \"signature\": \"11a87298deaf78b478c1a13abcfbfb1cc91f2540eb4fc5fd7daaf3a53b426cf8\"\n },\n {\n \"id\": \"ERR-H-L560-PARKED-DOMAINS-SWITCH-TO-DNS01-AFTER-HTTP01-FAILURES-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 560,\n \"title\": \"PARKED_DOMAINS_SWITCH_TO_DNS01_AFTER_HTTP01_FAILURES_20260701\",\n \"summary\": \"- Context: parked domains newfi.ru, hapusya.ru and kingofwolk.ru. - HTTP-01 attempts hit NPMplus default HTTP-to-HTTPS redirect/include-order problems. - Stage12 cleanup confirmed temp files 980-985 absent and nginx reload OK. - Rule: do not continue HTTP-01 for these parked domains; use DNS-01 with a dedicated Cloudflare token that has Zone Read and DNS Edit for the three zones.\",\n \"rule_like\": true,\n \"signature\": \"f4fbcb9b2153251384025a82debe093241f087f381f31e999e5de5c66b60d748\"\n },\n {\n \"id\": \"ERR-H-L566-PARKED-DOMAINS-STAGE15-FALSE-LOCAL-VALIDATE-ROLLBACK-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 566,\n \"title\": \"PARKED_DOMAINS_STAGE15_FALSE_LOCAL_VALIDATE_ROLLBACK_20260701\",\n \"summary\": \"- Context: Stage15 DNS-01 succeeded and public validation showed parked page for most hosts, but local edge curl --resolve validation returned CURL_RC=60 and triggered rollback. - Impact: do not rerun Stage15 as-is. - Rule: for parked domains finalization, use existing DNS-01 certificates and validate from pve01 public HTTPS with content marker; do not use the failed local --resolve TLS check as the closure gate.\",\n \"rule_like\": true,\n \"signature\": \"3b69e4a267b02bdaf0f69e8af7da2cf979ae13223acd5a1eed82f9de1736399d\"\n },\n {\n \"id\": \"ERR-H-L571-DOMAIN-ACTIONS-STAGE18-PLACEHOLDER-AND-RENEWAL-PROOF-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 571,\n \"title\": \"DOMAIN_ACTIONS_STAGE18_PLACEHOLDER_AND_RENEWAL_PROOF_20260701\",\n \"summary\": \"- Context: after parked domains and portfolio audit, operator requested removing domain enumeration from placeholder and checking certificate renewal paths. - Rule: Stage18 must not change DNS records or NPMplus route files. It may update placeholder HTML, run existing forum renewal script once for runtime proof, and run certbot dry-run for parked certificates using existing DNS-01 hooks.\",\n \"rule_like\": true,\n \"signature\": \"ff7fb700a503f6caecb02ad5f31ed4eefe5678fd9217ef289fc3c906cd291e70\"\n },\n {\n \"id\": \"ERR-H-L575-GRAM1-ROOT-WWW-PLACEHOLDER-CLOSE-STAGE19-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 575,\n \"title\": \"GRAM1_ROOT_WWW_PLACEHOLDER_CLOSE_STAGE19_20260701\",\n \"summary\": \"- Context: operator requested gram1.ru root/www to use the existing placeholder page. - Rule: only root gram1.ru and www.gram1.ru may be routed. Existing gram1.ru subdomain NPMplus routes must not be changed. - Rule: use DNS-01 with a dedicated gram1 Cloudflare token; do not use HTTP-01. - Stage18 parked dry-run note: kingofwolk.ru dry-run hit Let's Encrypt rateLimited/service-busy after active certificate issuance and public HTTPS were already OK. Treat as transient external dry-run failure, not route/cert closure blocker.\",\n \"rule_like\": true,\n \"signature\": \"9e2adc97128fdb5159168a0ebad49e1808253dcf358cdc908303a1a29d9277b8\"\n },\n {\n \"id\": \"ERR-H-L581-PVEPRO-EDGE-LANDING-STAGE21-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 581,\n \"title\": \"PVEPRO_EDGE_LANDING_STAGE21_20260701\",\n \"summary\": \"- Context: pvepro.ru root/www currently point to the Mailcow VPS and return Mailcow with TLS hostname mismatch. - Decision: because pve01 has no SSH key access to the external VPS, do not modify the VPS. Instead move only pvepro.ru and www.pvepro.ru A records to edge and serve an edge landing page. - Safety rule: do not change mail.pvepro.ru, nb.pvepro.ru, MX, SPF, DKIM, DMARC or forum SMTP configuration. - Use a dedicated Cloudflare token for pvepro.ru. Do not print token values.\",\n \"rule_like\": true,\n \"signature\": \"da430a8849bebcbd0a5dc55d4282ae916ffb348af757e797b50a144a35c00e29\"\n },\n {\n \"id\": \"ERR-H-L587-PVEPRO-STAGE21-LANDING-SERVICE-FAILED-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 587,\n \"title\": \"PVEPRO_STAGE21_LANDING_SERVICE_FAILED_20260701\",\n \"summary\": \"- Context: Stage21 stopped safely before DNS/cert/NPMplus route because local pvepro landing service returned CURL_RC=7 on 127.0.0.1:18089. - Impact: DNS root/www pvepro.ru remained on 185.139.214.215 and no route/cert changes were applied. - Rule: Stage22 must fix and prove the local landing service first, then proceed to DNS-01 cert, NPMplus route and root/www DNS only after local marker PVEPRO_LANDING_OK is reachable.\",\n \"rule_like\": false,\n \"signature\": \"6c8c70fd8a9af6ca28d7c2848c52dae8dd0405dfb2758b3341fb7d3c0f24131f\"\n },\n {\n \"id\": \"ERR-H-L592-PVEPRO-STAGE22-VALIDATION-AND-COPY-GUARD-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 592,\n \"title\": \"PVEPRO_STAGE22_VALIDATION_AND_COPY_GUARD_20260701\",\n \"summary\": \"- Context: Stage22 succeeded on edge-side landing, cert, NPMplus route and Cloudflare A upserts, but wrapper had a non-critical scp wildcard failure due __pycache__ and public validation used cached DNS for root pvepro.ru. - Mistakes recorded: do not scp scripts/* after local py_compile because __pycache__ may be present; do not continue after critical scp failure in future apply stages; do not classify PVEPro landing as NetBird just because it contains a link text; use Cloudflare authoritative A records and --resolve validation when DNS propagation may lag. - Stage23 must be read-only for infrastructure and may close only if edge route, Cloudflare A records, HTTPS landing marker, and mail/nb unchanged checks pass.\",\n \"rule_like\": true,\n \"signature\": \"39b7838d47fba86c663aac0896edc9db58fa614ebdcdbae09fb3054a061ba44f\"\n },\n {\n \"id\": \"ERR-H-L597-TAFTAUTO-CERT-AUTORENEW-BLOCKED-NO-PRIVATE-ACCESS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 597,\n \"title\": \"TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701\",\n \"summary\": \"- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router. - Current limitation: there is no safe private remote access path to the dacha router yet. - Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet. - Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists.\",\n \"rule_like\": true,\n \"signature\": \"9cd06e064f5a86be9626c4f9a2a819b8d0e71b03309c705fe0e34e1e59f81b73\"\n },\n {\n \"id\": \"ERR-H-L603-TAFTAUTO-CERT-AUTORENEW-BLOCKED-NO-PRIVATE-ACCESS-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 603,\n \"title\": \"TAFTAUTO_CERT_AUTORENEW_BLOCKED_NO_PRIVATE_ACCESS_20260701\",\n \"summary\": \"- Context: taftauto.ru points to the dacha router public IP. Operator confirmed the router model is Netcraze-like, same family as Moscow router. - Current limitation: there is no safe private remote access path to the dacha router yet. - Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet. - Closure status: certificate autodeploy is intentionally blocked until VPN/NetBird/WireGuard or another private management path exists.\",\n \"rule_like\": true,\n \"signature\": \"9cd06e064f5a86be9626c4f9a2a819b8d0e71b03309c705fe0e34e1e59f81b73\"\n },\n {\n \"id\": \"ERR-H-L609-TAFTAUTO-WG-STAGE24-SECRET-AND-IMPORT-RULE-20260701\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 609,\n \"title\": \"TAFTAUTO_WG_STAGE24_SECRET_AND_IMPORT_RULE_20260701\",\n \"summary\": \"- Context: configuring private management path for taftauto.ru dacha router. - Rule: do not print WireGuard private keys, PSK, or client config body in proof/chat. - Rule: generated client config may be stored only as root/debian mode 600 file and must be imported into the Netcraze router UI manually. - Rule: do not open router public admin or certificate deployment until WireGuard handshake is proven.\",\n \"rule_like\": true,\n \"signature\": \"6722ad8ce177f600b198fb48046657acb926330bfbd8bbe1c810de69015f5a87\"\n },\n {\n \"id\": \"ERR-H-L615-TAFTAUTO-WG-PSK-ROTATION-SCRIPT-BROKE-DATAPLANE-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 615,\n \"title\": \"TAFTAUTO_WG_PSK_ROTATION_SCRIPT_BROKE_DATAPLANE_20260702\",\n \"summary\": \"- Assistant gave unsafe PSK rotation flow; after profile reimport dacha interface changed from Wireguard0 to Wireguard1 and data-plane broke until security-level, ACL and routes were restored on Wireguard1. - Proof 666 is superseded because it showed PUBLIC_SSH_22_STILL_OPEN. - Final fixed proof: /root/evidence/667_TAFTAUTO_WG_PSK_ROTATED_PUBLIC_CLOSED_OK_20260702_PROOF.txt\",\n \"rule_like\": false,\n \"signature\": \"4b046c53fa6020f8c97bd7e00a8a7d78280d3ab2bf337816cb559083ba9c7495\"\n },\n {\n \"id\": \"ERR-H-L620-TAFTAUTO-CERTBOT-CLOUDFLARE-PLUGIN-MISSING-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 620,\n \"title\": \"TAFTAUTO_CERTBOT_CLOUDFLARE_PLUGIN_MISSING_20260702\",\n \"summary\": \"- Certbot on edge rejected --dns-cloudflare-* arguments: dns-cloudflare plugin missing/not loaded. - No taftauto certificate issued in failed run; no DNS A records or NPMplus routes changed.\",\n \"rule_like\": false,\n \"signature\": \"3e7ad3e9ef090a52212334ce2d8c5b65fa7dae68481c01068ec7845bef14a07f\"\n },\n {\n \"id\": \"ERR-H-L624-HOMEPAGE-SERVICES-YAML-BAD-INDENT-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 624,\n \"title\": \"HOMEPAGE_SERVICES_YAML_BAD_INDENT_20260702\",\n \"summary\": \"- Assistant inserted Homepage services.yaml entries with wrong indentation; YAMLException bad indentation at line 227. - Fix: rebuild SmartApe under External Homelabs with 2-space item indentation, remove Router Moscow, rebuild Sites category.\",\n \"rule_like\": false,\n \"signature\": \"da1ecebf560240f444c3df39efb068b9d7c8ab7ceda2053516e8d82c8567bd7b\"\n },\n {\n \"id\": \"ERR-H-L628-HOMEPAGE-SERVICES-YAML-BAD-INDENT-REPAIR-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 628,\n \"title\": \"HOMEPAGE_SERVICES_YAML_BAD_INDENT_REPAIR_20260702\",\n \"summary\": \"- Previous apply broke services.yaml indentation and did not follow YAML-aware rule. - Repair uses latest valid services.yaml backup, PyYAML parse/dump, validates before restart.\",\n \"rule_like\": false,\n \"signature\": \"7b789067b17326a4ead14942608a43b86b4ae971896eff93eb8658684e657e22\"\n },\n {\n \"id\": \"ERR-H-L632-TAFTAUTO-CERTBOT-DRYRUN-BAD-FLAG-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 632,\n \"title\": \"TAFTAUTO_CERTBOT_DRYRUN_BAD_FLAG_20260702\",\n \"summary\": \"- Assistant used unsupported certbot flag --manual-public-ip-logging-ok with certbot 4.0.0 during taftauto renew dry-run. - No certificate, DNS, NPMplus route, or deploy state changed by the failed dry-run.\",\n \"rule_like\": false,\n \"signature\": \"518dbc241694b2eb7c4631a1b147a8262af537e6f4eaf082d1f092cc0a8087cb\"\n },\n {\n \"id\": \"ERR-H-L636-TAFTAUTO-CERT-RENEW-DRYRUN-RATE-LIMITED-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 636,\n \"title\": \"TAFTAUTO_CERT_RENEW_DRYRUN_RATE_LIMITED_20260702\",\n \"summary\": \"- Proof 672 is superseded/partial: it printed STATUS=OK even though certbot dry-run failed. - Failure reason: Let's Encrypt staging/service returned rateLimited / Service busy; retry later. - Cloudflare manual auth and cleanup hooks did run successfully. - Deploy hook was installed and manually invoked successfully before the dry-run. - Do not retry immediately.\",\n \"rule_like\": true,\n \"signature\": \"15452c7cc7d791a7cc8324a5f2d39544b8a0971fce736a46afdfce7ec8eea5b3\"\n },\n {\n \"id\": \"ERR-H-L643-HOMEPAGE-XENFORO-INFO-INLINE-PYTHON-SYNTAX-ERROR-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 643,\n \"title\": \"HOMEPAGE_XENFORO_INFO_INLINE_PYTHON_SYNTAX_ERROR_20260702\",\n \"summary\": \"- Assistant used invalid inline Python syntax with def after semicolon while adding XenForo.info to Homepage External. - Failed before services.yaml write.\",\n \"rule_like\": false,\n \"signature\": \"3620b01df453a95e2e7fbd121ee954690344e32ed12c71b84739e1a554b316d2\"\n },\n {\n \"id\": \"ERR-H-L647-HOMEPAGE-USEFUL-ROUTER-NAME-MISMATCH-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 647,\n \"title\": \"HOMEPAGE_USEFUL_ROUTER_NAME_MISMATCH_20260702\",\n \"summary\": \"- Previous apply looked for Russian Router card name, but active Homepage Useful group contains Router and Public Domain. - Failed before write. Fix uses YAML-aware edit with exact names Router/Public Domain.\",\n \"rule_like\": false,\n \"signature\": \"49212250bc5eea53887790a1782ea3acdc2fa2e7769d89ee79f603883533adf8\"\n },\n {\n \"id\": \"ERR-H-L651-HOMEPAGE-USEFUL-ROUTER-CARD-NOT-FOUND-20260702\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 651,\n \"title\": \"HOMEPAGE_USEFUL_ROUTER_CARD_NOT_FOUND_20260702\",\n \"summary\": \"- Apply expected Useful -> Router in active services.yaml, but YAML parse did not find it. No write was performed. - Need verify active Homepage config path, docker mount, and any alternate services.yaml before next apply.\",\n \"rule_like\": false,\n \"signature\": \"ce9a93f7876b27d64e76cddc59b220166fa0e82ab7265d3373352906d6e10264\"\n },\n {\n \"id\": \"ERR-H-L655-20260702-CF-TOKEN-AUDIT-BROKEN-COMMAND\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 655,\n \"title\": \"20260702_CF_TOKEN_AUDIT_BROKEN_COMMAND\",\n \"summary\": \"- Ошибка: ассистент дал хрупкую команду аудита Cloudflare-токенов с вложенным heredoc/Python через ssh. - Факт: bash начал интерпретировать Python-код, появились syntax error и вторичный prompt >. - Правило: не давать nested heredoc/сложные кавычки через ssh; для длинной логики сначала класть скрипт отдельным файлом, затем запускать его. - Статус: аудит токенов не выполнен, инфраструктурные изменения не должны считаться применёнными.\",\n \"rule_like\": false,\n \"signature\": \"00c3ff174b7bd8af54a071aad893fc3e28aae0f79b8cba6babffa67869d0d7c3\"\n },\n {\n \"id\": \"ERR-H-L661-LESSON-20260702-CROWDSEC-NETBIRD-EXIT-ROUTE\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 661,\n \"title\": \"LESSON_20260702_CROWDSEC_NETBIRD_EXIT_ROUTE\",\n \"summary\": \"- Do not treat e3qxxx.netbird.selfhosted / 100.100.125.70 and e3qxxx-183-106.netbird.selfhosted / 100.100.183.106 as VPS egress nodes; they are user mobile peers. - Correct egress candidates are relay.pvepro.ru / relay.netbird.selfhosted / 100.100.19.1 for Moldova and mail.pvepro.ru / mail.netbird.selfhosted / 100.100.147.204 for USA. - Access Policy edge-vm -> relay/mail only proves peer reachability. It is not an internet exit route. - Required NetBird fix for CrowdSec CAPI was Network Routing exit route 0.0.0.0/0 distributed to edge-vm group, routing peer relay, masquerade/NAT enabled. - Before attempting CrowdSec CAPI, verify on edge-vm: ip route get 1.1.1.1 uses wt0, wg allowed-ips contains 0.0.0.0/0, and Cloudflare trace no longer shows home IP 95.84.154.183.\",\n \"rule_like\": true,\n \"signature\": \"651aa75a20b1bad649519711690f3a5d4f944e71649c31414c966c56c2647f83\"\n },\n {\n \"id\": \"ERR-H-L668-LESSON-20260702-HOMEPAGE-MONITORS-REPAIR-NOT-DELETE\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 668,\n \"title\": \"LESSON_20260702_HOMEPAGE_MONITORS_REPAIR_NOT_DELETE\",\n \"summary\": \"- Do not delete or disable Homepage siteMonitor fields to hide red badges. - Fix the underlying monitor path, DNS, proxy route, or local health-wrapper instead. - Do not touch Cloudflare Homepage card when operator says it is green and opens correctly. - Before changing Homepage, identify the live config host; in this state live Homepage is on core-apps [PRIVATE_IP], not edge-vm.\",\n \"rule_like\": true,\n \"signature\": \"2bc16e142d368e59b2b176de6779445c152df7e8bae2b1be79a2f4d075985c7c\"\n },\n {\n \"id\": \"ERR-H-L674-LESSON-20260702-KUMA-ADD-ONE-MONITOR-ONLY\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 674,\n \"title\": \"LESSON_20260702_KUMA_ADD_ONE_MONITOR_ONLY\",\n \"summary\": \"- When asked to fix one missing Uptime Kuma service, add only one monitor, then generate proposals separately. - Before direct Kuma DB mutation, stop the container and create a DB backup. - Verify DB integrity before starting Kuma again. - Do not touch Cloudflare when operator says it is green and opens correctly.\",\n \"rule_like\": true,\n \"signature\": \"b268d57842a9dd192fd72dd940879f3e7cf714ae2928a68a23b5e1ed347262f0\"\n },\n {\n \"id\": \"ERR-H-L680-LESSON-20260702-DOCKGE-STALE-STACKS-NOT-CONTAINERS\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 680,\n \"title\": \"LESSON_20260702_DOCKGE_STALE_STACKS_NOT_CONTAINERS\",\n \"summary\": \"- Dockge inactive items after migration can be stale compose folders, not stopped containers. - First classify runtime projects across all Docker hosts before deleting or archiving anything. - For remote Dockge visibility, projects must be under /opt/stacks or bind-mounted there. - Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions.\",\n \"rule_like\": true,\n \"signature\": \"4249f24d95f2186959586b18fa083cdac97ceddb1cc65b0bd0fd1eeddfdea50d\"\n },\n {\n \"id\": \"ERR-H-L686-STAGE4B-MASKED-SQL-QUERY-FAILURE-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 686,\n \"title\": \"STAGE4B_MASKED_SQL_QUERY_FAILURE_20260714\",\n \"summary\": \"- Previous Stage4B read-only audit returned empty observation-summary, metadata-key, trigger and function blocks because nested SSH/SQL quoting broke queries while stderr was hidden. - Empty blocks are query failures, not proof that metadata, triggers or functions are absent. - Future PostgreSQL audits must use ON_ERROR_STOP, visible stderr, independent query RC and quote-safe dollar literals.\",\n \"rule_like\": false,\n \"signature\": \"9786689ba74fed176972e94389a37a7664ee81618201ff24852199ed754b694f\"\n },\n {\n \"id\": \"ERR-H-L691-STAGE4C-SCHEMA-MIGRATIONS-ID-ASSUMPTION-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 691,\n \"title\": \"STAGE4C_SCHEMA_MIGRATIONS_ID_ASSUMPTION_20260714\",\n \"summary\": \"- Stage4C read-only preflight incorrectly assumed schema_migrations has a column named id. - PostgreSQL returned column id does not exist and ON_ERROR_STOP correctly stopped all following SQL checks. - Production DB and application were not changed. Future migration audits must inspect information_schema first and query migration rows without assuming column names.\",\n \"rule_like\": false,\n \"signature\": \"f166dab2251442df1f0e77fb70dfa166f1e681a94567cc0d17eb7e5000466a11\"\n },\n {\n \"id\": \"ERR-H-L696-STAGE4C-PGDUMP-DEV-NULL-FSYNC-INVALID-PROBE-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 696,\n \"title\": \"STAGE4C_PGDUMP_DEV_NULL_FSYNC_INVALID_PROBE_20260714\",\n \"summary\": \"- Stage4C custom-format pg_dump capability probe incorrectly used /dev/null as the output file. - pg_dump failed only because fsync on /dev/null returned Invalid argument. - Production database and application were not changed. - Future custom dump capability checks must use a real temporary regular file, validate it with pg_restore --list, and remove it afterward.\",\n \"rule_like\": false,\n \"signature\": \"c565b0d0dc04089e68783eb6f6b52e7e66fd66b73eb84b37ea6d7f324ca2142c\"\n },\n {\n \"id\": \"ERR-H-L702-STAGE4C-REMOTE-PREP-THIRD-SUBSTEP-FAILED-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 702,\n \"title\": \"STAGE4C_REMOTE_PREP_THIRD_SUBSTEP_FAILED_20260714\",\n \"summary\": \"- Isolated migration dry-run did not start: REMOTE_PREP_RC=0|0|1. - Remote directory creation and SCP succeeded; the third ownership/mode preparation substep returned RC=1. - Production database and application were not changed. - Exact cause requires read-only residual directory and permission inspection before cleanup or retry.\",\n \"rule_like\": false,\n \"signature\": \"c048e5da3ee1467f7b4742ae2ef671944078c985979c4c75f87aefe2b13c54ad\"\n },\n {\n \"id\": \"ERR-H-L708-STAGE4C-REMOTE-PREP-GLOB-AFTER-CHMOD-ROOT-CAUSE-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 708,\n \"title\": \"STAGE4C_REMOTE_PREP_GLOB_AFTER_CHMOD_ROOT_CAUSE_20260714\",\n \"summary\": \"- Exact cause of REMOTE_PREP_RC=0|0|1: after the directory became postgres-owned mode 0700, the unprivileged debian shell could not traverse it and could not expand the wildcard used by chmod. - Candidate files and SHA256 verification were valid; no temporary database was created and production was unchanged. - Future preparation must use exact privileged paths or a verified shared group, never an unprivileged wildcard after restrictive chmod.\",\n \"rule_like\": true,\n \"signature\": \"a174131243eb8db72dedb89bc3e917cb45e7c7c8107e40c90b476d46bf4d2718\"\n },\n {\n \"id\": \"ERR-H-L713-STAGE4C-MIGRATION003-FINGERPRINT-DOLLAR-QUOTE-FAILURE-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 713,\n \"title\": \"STAGE4C_MIGRATION003_FINGERPRINT_DOLLAR_QUOTE_FAILURE_20260714\",\n \"summary\": \"- Migration 003 isolated dry-run failed at fingerprint CHECK because the regex end anchor collided with the SQL dollar-quote delimiter and produced an unterminated dollar-quoted string. - The migration transaction rolled back, the temporary database was removed, and production remained unchanged. - Fingerprint validation must use length plus translate without a regex dollar anchor; corrected candidate must pass a fresh isolated dry-run before production consideration.\",\n \"rule_like\": false,\n \"signature\": \"adc9a3f845fb0a58a7d9e0196174f3a9c0b5eb905bbd2c2eb456a92eb9f2d37b\"\n },\n {\n \"id\": \"ERR-H-L718-STAGE4C-SEAL-OUTER-RC-MASKING-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 718,\n \"title\": \"STAGE4C_SEAL_OUTER_RC_MASKING_20260714\",\n \"summary\": \"- Stage4C final seal correctly refused creation because overall health was WARN, returning internal RC=40. - The former wrapper then printed a second misleading COMMAND_RC=0 because top-level true masked the internal result. - Future commands must capture the guarded subshell through PIPESTATUS and print exactly one authoritative COMMAND_RC before the mandatory final true. - Production database, application and services were unchanged.\",\n \"rule_like\": false,\n \"signature\": \"9f71003c157467e9b2bc0405d3137b6f0f7d14e053cb3053e9323219494287a9\"\n },\n {\n \"id\": \"ERR-H-L724-STAGE4C-DIAGNOSIS-EXPECTED-DERIVED-FILE-ON-VM180-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 724,\n \"title\": \"STAGE4C_DIAGNOSIS_EXPECTED_DERIVED_FILE_ON_VM180_20260714\",\n \"summary\": \"- Previous diagnosis incorrectly treated cluster-admin-restricted-probes.txt as a required source file on VM180. - The file is a central derived health artifact on pve01; VM180 supplies full-observer evidence instead. - This false expectation caused diagnostic RC=32 without any infrastructure failure or production change.\",\n \"rule_like\": false,\n \"signature\": \"d032d83a642f525e4ef82024ba9f810d8b9cbb88a6351da3dbed17391e14056d\"\n },\n {\n \"id\": \"ERR-H-L729-STAGE4C-NO-JOBS-RUNNING-TEXT-COUNTED-AS-JOB-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 729,\n \"title\": \"STAGE4C_NO_JOBS_RUNNING_TEXT_COUNTED_AS_JOB_20260714\",\n \"summary\": \"- Stage4C seal preflight incorrectly counted the systemctl informational text No jobs running as one active job. - The parser counted every nonempty output line instead of accepting only rows beginning with a numeric systemd Job ID. - No service was started and no production state changed during the blocked attempt. - Future job counts must match a numeric first field only.\",\n \"rule_like\": false,\n \"signature\": \"619b903f2e18ba4251959e200fe4511995e56c4cd9a3e63b5fc8d77134df5ade\"\n },\n {\n \"id\": \"ERR-H-L735-STAGE4D-ADAPTER-SOURCE-AUDIT-ABORTED-BEFORE-FINAL-MARKERS-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 735,\n \"title\": \"STAGE4D_ADAPTER_SOURCE_AUDIT_ABORTED_BEFORE_FINAL_MARKERS_20260714\",\n \"summary\": \"- The evidence-only Stage4D adapter source audit returned RC=1 before emitting its VM180 completion and final audit markers. - The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults. - Production, database, application, services, timers, health and desired-state were unchanged. - Inspect the preserved runner stdout, stderr and result JSON before correcting or rerunning the task.\",\n \"rule_like\": true,\n \"signature\": \"cc2c72b5d6eb7a0a1f3258050f4509d2e08f538236449d0100a9ee45193da58d\"\n },\n {\n \"id\": \"ERR-H-L741-STAGE4D-LOCAL-EXPANSION-OF-REMOTE-Q-UNDER-NOUNSET-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 741,\n \"title\": \"STAGE4D_LOCAL_EXPANSION_OF_REMOTE_Q_UNDER_NOUNSET_20260714\",\n \"summary\": \"- The first Stage4D adapter source audit embedded a large remote program inside a locally double-quoted SSH argument. - Local shell expansion reached the remote variable Q while nounset was enabled, causing Q: unbound variable before SSH execution. - VM180 and PostgreSQL audits did not start; production state was unchanged. - Remote shell and SQL programs must be separate syntax-checked files passed through SSH stdin.\",\n \"rule_like\": false,\n \"signature\": \"bdae2b3ceda91c697838a15c35c327d73583cc201d5fc77c2f7355f45c2e340a\"\n },\n {\n \"id\": \"ERR-H-L747-STAGE4D-REMOTE-PYTHON311-ADAPTER-SELFTEST-RC1-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 747,\n \"title\": \"STAGE4D_REMOTE_PYTHON311_ADAPTER_SELFTEST_RC1_20260714\",\n \"summary\": \"- Canonical adapter candidate passed local Python 3.13 self-tests with 48 envelopes but returned RC=1 on VM180 Python 3.11.2. - Remote upload and cleanup succeeded, and production database remained 0|0|OK. - The exact traceback and failing assertion must be captured by an isolated identical reproduction before changing the candidate.\",\n \"rule_like\": false,\n \"signature\": \"6111533bbd96a60d54d47dff09af497c706e31b90977efb40efc8d5c24bf5ada\"\n },\n {\n \"id\": \"ERR-H-L752-STAGE4D-PYTHON311-FSTRING-COMPATIBILITY-AND-REPRO-VALIDATOR-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 752,\n \"title\": \"STAGE4D_PYTHON311_FSTRING_COMPATIBILITY_AND_REPRO_VALIDATOR_20260714\",\n \"summary\": \"- Adapter candidate passed Python 3.13 but failed Python 3.11 parsing at Path(row[\\\"path\\\"]) inside a double-quoted f-string. - Exact fix is Python 3.11-compatible quoting: Path(row['path']). - The reproduction command captured the expected compile failure but incorrectly returned CAPTURE_BAD=1 because its validator required compile RC=0. - Production database remained 0|0|OK and desired-state remained clean. - Future adapter candidates must pass syntax and full self-tests on both pve01 and VM180 runtimes before acceptance.\",\n \"rule_like\": false,\n \"signature\": \"5aec931e3d68bbdfd0f84e397c8ad1aea4fe7e9d239f066d7186823aaf5a0c33\"\n },\n {\n \"id\": \"ERR-H-L759-STAGE4E-DESIGN-VALIDATOR-ABORTED-BEFORE-LOCAL-VALIDATION-MARKER-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 759,\n \"title\": \"STAGE4E_DESIGN_VALIDATOR_ABORTED_BEFORE_LOCAL_VALIDATION_MARKER_20260714\",\n \"summary\": \"- Restricted probe-agent design candidate compiled successfully but its local validator exited before producing LOCAL_VALIDATION_RC. - The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults. - Active external probes were not executed and production state was unchanged. - Inspect the preserved validator traceback and exact assertion before modifying the candidate.\",\n \"rule_like\": false,\n \"signature\": \"fa47b366640ad2b68b1efc2d897808d7e75322d7aa9e3c9af89013e4eecc5a26\"\n },\n {\n \"id\": \"ERR-H-L765-STAGE4E-VALIDATOR-SCANNED-PYC-AND-OWN-FORBIDDEN-LITERALS-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 765,\n \"title\": \"STAGE4E_VALIDATOR_SCANNED_PYC_AND_OWN_FORBIDDEN_LITERALS_20260714\",\n \"summary\": \"- Stage4E design validator recursively read every candidate file as UTF-8 after py_compile had created binary __pycache__ bytecode. - This caused UnicodeDecodeError before any design assertion failed. - The phase-wide forbidden-pattern scan also included validate_design.py itself, which necessarily contains the prohibited literals it verifies. - Validators must use an explicit text-file allowlist and scans must exclude validator implementation and binary artifacts. - Production, database, services, timers and desired-state were unchanged; active external probes were not executed.\",\n \"rule_like\": true,\n \"signature\": \"76b2888a1b774c0f7e8c0fe3aab0e3beb73c78f9b46ef3694f4708cc3431230c\"\n },\n {\n \"id\": \"ERR-H-L772-STAGE4E-V2-VALIDATOR-FAILED-STATIC-COMPILE-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 772,\n \"title\": \"STAGE4E_V2_VALIDATOR_FAILED_STATIC_COMPILE_20260714\",\n \"summary\": \"- Stage4E design candidate v2 failed static compilation before local design validation started. - The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects. - The generated validator must be inspected at the exact SyntaxError line before another candidate is created. - Production, database, application, services, timers, health and desired-state were unchanged; active external probes were not executed.\",\n \"rule_like\": false,\n \"signature\": \"0acd318ec69af959e23233f89a6b26cefcce1dbb2c85908fcc2406c51c85db72\"\n },\n {\n \"id\": \"ERR-H-L778-STAGE4E-PIPESTATUS-LOST-AFTER-FIRST-ASSIGNMENT-20260714\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 778,\n \"title\": \"STAGE4E_PIPESTATUS_LOST_AFTER_FIRST_ASSIGNMENT_20260714\",\n \"summary\": \"- VM180 validation copy used a pipeline and read PIPESTATUS in two separate assignments. - The first assignment reset PIPESTATUS, leaving COPY_RC empty and preventing remote validation and cleanup. - Retry must use explicit SCP operations without pipeline status parsing. - Production remained unchanged and active external probes were not executed.\",\n \"rule_like\": false,\n \"signature\": \"29d5bdfa098352aeceb9defce46f9845d100c46dfc1758f17ad8c5eb56859787\"\n },\n {\n \"id\": \"ERR-H-L784-STAGE4F-VALIDATOR-EXPECTED-HEALTH-REFRESH-FROM-NONWRITING-BACKUP-SCRIPT-\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 784,\n \"title\": \"STAGE4F_VALIDATOR_EXPECTED_HEALTH_REFRESH_FROM_NONWRITING_BACKUP_SCRIPT_20260714\",\n \"summary\": \"- The controlled backup service completed with Result=success and ExecMainStatus=0. - Validation incorrectly required cluster-admin-incident-engine-backup.txt to receive a fresh timestamp. - The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure. - Stage4F backup proof must use the service result, fresh latest JSON, produced artifacts and off-host/restore evidence instead. - The backup must not be rerun merely to satisfy the invalid health-file freshness assertion.\",\n \"rule_like\": true,\n \"signature\": \"92516bc4ce3841e809e6ec04f2ec77458c1c472c146af37a577bbeffbce5d19d\"\n },\n {\n \"id\": \"ERR-H-L791-STRICT-RULE-20260714-CLOSE-TAILS-IMMEDIATELY\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 791,\n \"title\": \"STRICT_RULE_20260714_CLOSE_TAILS_IMMEDIATELY\",\n \"summary\": \"- Failure class: переход к следующей задаче при наличии незакрытого хвоста. - Mandatory anti-regression: после ошибки запрещено продолжать другой scope до точной root cause, исправления, повторной проверки, очистки, proof и seal. - Closure gate: UNRESOLVED_TAIL_COUNT=0; BLOCKER_COUNT=0; TEMPORARY_ARTIFACT_COUNT=0; ROLLBACK_VERIFIED=YES; HEALTH=OK; PROOF_UPDATED=YES; REFERENCE_UPDATED=YES. - Если закрытие невозможно из-за внешней зависимости, опасной неоднозначности или обязательного решения пользователя, фиксировать BLOCKED/OPEN и не заявлять CLOSED. - Proof: /var/lib/homelab-change-sets/operator-rule-close-tails-immediately-v1/20260714T171631Z/report.txt\",\n \"rule_like\": true,\n \"signature\": \"951ac14651bd989701f8f0cdf4a6f683fdbe957a357523c878cef0081fdc8dd7\"\n },\n {\n \"id\": \"ERR-H-L798-ERROR-20260714-DEPENDENCY-FACT-QUERY-GLOBAL-LIMIT\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 798,\n \"title\": \"ERROR_20260714_DEPENDENCY_FACT_QUERY_GLOBAL_LIMIT\",\n \"summary\": \"- Symptom: dependency audit вернул только latest_collector_status. - Root cause: ORDER BY 1 LIMIT 1 в конце UNION ALL применился ко всему набору фактов. - Correction: latest collector status извлекается скалярным подзапросом внутри CTE facts; глобальный LIMIT отсутствует. - Anti-regression: multi-fact SQL audit обязан проверять точное ожидаемое число строк до извлечения значений. - Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt - Status: CLOSED.\",\n \"rule_like\": false,\n \"signature\": \"8078498e8c9915662cb7bbe485ccb1e71099a6a8d39849830721e2bcd69fde70\"\n },\n {\n \"id\": \"ERR-H-L806-ERROR-20260714-COLLECTOR-PATCH-REQUIREMENT-PATTERN-MISMATCH\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 806,\n \"title\": \"ERROR_20260714_COLLECTOR_PATCH_REQUIREMENT_PATTERN_MISMATCH\",\n \"summary\": \"- Symptom: MIGRATION_PLAN_COLLECTOR_PATCH_REQUIRED_COUNT=1 при ожидаемых двух подтверждениях. - Root cause: узкий шаблон учитывал collector_patch_required, но не collector_patch_required_after_stage4c. - Correction: оба поля проверяются отдельно, затем складываются как semantic requirement count. - Anti-regression: связанные JSON-контракты проверяются по собственным точным именам полей, а не одним унифицированным grep-шаблоном. - Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-provenance-readiness-scope-v1/report.txt - Status: CLOSED; PRIOR_TAIL_UNRESOLVED_COUNT=0.\",\n \"rule_like\": false,\n \"signature\": \"a136bb3af68dd9abefd3054a299cbe72d1ee4435d755299318badd0469472379\"\n },\n {\n \"id\": \"ERR-H-L814-ERROR-20260714-PRIVILEGED-FILE-REDIRECTION-BEFORE-SUDO\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 814,\n \"title\": \"ERROR_20260714_PRIVILEGED_FILE_REDIRECTION_BEFORE_SUDO\",\n \"summary\": \"- Symptom: bash reported Permission denied while counting collector.py lines. - Root cause: `< collector.py` was opened by the unprivileged caller shell before sudo executed wc. - Correction: run sudo wc -l collector.py without caller-side input redirection. - Anti-regression: never combine sudo with `< protected-file`; pass the protected path as an argument to the privileged command. - Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt - Status: CLOSED.\",\n \"rule_like\": true,\n \"signature\": \"4510dad88d4639e1987b83002657a7c5c4580d0b4ae4b398c376b2ca49aa394d\"\n },\n {\n \"id\": \"ERR-H-L822-ERROR-20260714-COLLECTOR-RUN-COLUMN-ASSUMPTION\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 822,\n \"title\": \"ERROR_20260714_COLLECTOR_RUN_COLUMN_ASSUMPTION\",\n \"summary\": \"- Symptom: SQL failed because completed_at did not exist. - Root cause: collector_runs columns were assumed instead of read from information_schema; actual names are finished_at and error_text. - Correction: assert required and forbidden column counts before querying recent runs. - Anti-regression: every schema-sensitive audit must discover and validate exact column names before the main query. - Closure proof: /var/lib/homelab-cluster-admin-stage4/stage4h-collector-baseline-audit-final-v1/report.txt - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": true,\n \"signature\": \"01f861530e0df5cbb6814d21fe0caa2043d2819342012f74d4299c45bfcf08cd\"\n },\n {\n \"id\": \"ERR-H-L830-ERROR-20260714-REMOTE-STDIN-ARGUMENT-SHIFT\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 830,\n \"title\": \"ERROR_20260714_REMOTE_STDIN_ARGUMENT_SHIFT\",\n \"summary\": \"- Symptom: remote harness выполнил chmod для пути bash. - Root cause: дополнительный positional token bash был передан после SSH command и стал первым аргументом stdin-скрипта. - Correction: remote command вызывает bash -s -- с ровно тремя явно встроенными аргументами: candidate path, SHA256 и byte count. - Anti-regression: stdin-скрипт обязан валидировать $1 по разрешённому path pattern и затем подтверждать точные SHA256 и byte count; запрещено добавлять отдельный bash-token после remote command. - Closure proof: /var/lib/homelab-change-sets/stage4h-collector-provenance-patch-candidate-v1/20260714T174131Z/report.txt - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": true,\n \"signature\": \"a808754bcc91814fedf38578e86631a14e4964c3ab9571e205b6b65afa951389\"\n },\n {\n \"id\": \"ERR-H-L838-ERROR-20260714-OVERSIZED-COMMAND-PARSE-FAILURE\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 838,\n \"title\": \"ERROR_20260714_OVERSIZED_COMMAND_PARSE_FAILURE\",\n \"summary\": \"- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`. - Root cause: чрезмерно длинная интерактивная команда была обрезана или повреждена при передаче до полного разбора Bash. - Impact: side-effect audit подтвердил production state 0|0|OK, неизменный collector, активный timer и отсутствие временных БД, каталогов и процессов. - Correction: монолитные команды запрещены; сложные действия выполняются через отдельный проверяемый task-скрипт и короткий launcher. - Anti-regression: interactive payload target <= 8000 bytes; syntax-check перед запуском; remote output сохраняется независимо от RC. - Closure proof: /var/lib/homelab-change-sets/operator-command-size-guard-v1/20260714T182653Z/report.txt - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": false,\n \"signature\": \"16b0958121cab8d184e16e7ea30930971bc785f4385caaeacea19bb35a8e8908\"\n },\n {\n \"id\": \"ERR-H-L847-ERROR-20260714-HOMELAB-ADMIN-HELP-RC-ASSUMPTION\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 847,\n \"title\": \"ERROR_20260714_HOMELAB_ADMIN_HELP_RC_ASSUMPTION\",\n \"summary\": \"- Symptom: template audit завершился HOMELAB_ADMIN_HELP_FAILED. - Root cause: был ошибочно ожидаем RC=0 от неподдерживаемого аргумента --help. - Actual contract: неизвестный аргумент печатает usage и возвращает RC=64. - Correction: проверять version либо считать точную usage-строку вместе с RC=64 штатным результатом. - Anti-regression: перед проверкой CLI сначала фиксировать поддерживаемые команды; не предполагать GNU-style --help. - Closure proof: /var/lib/homelab-change-sets/homelab-admin-cli-contract-fix-v1/20260714T183338Z/report.txt - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": false,\n \"signature\": \"346a85cc8494d14ed401b27e57d0b73a8d53078fdd91575ccb4dc89e833d4e44\"\n },\n {\n \"id\": \"ERR-H-L856-ERROR-20260715-EXPECTED-NEGATIVE-RC-TRAP-CLASSIFICATION\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 856,\n \"title\": \"ERROR_20260715_EXPECTED_NEGATIVE_RC_TRAP_CLASSIFICATION\",\n \"summary\": \"- Symptom: schema-v2 task v3 build stopped while intentionally testing the --invalid argument path. - Root cause: an expected RC64 was executed while the generic ERR trap remained active. - Correction: expected failures are evaluated only through an explicit if/else branch with immediate RC capture. ANTI_REGRESSION_TOKEN=EXPECTED_RC64_INTERCEPTED_ERR_TRAP - Required pattern: if command; then rc=0; else rc=$?; fi, followed by a separate exact expected-RC assertion. - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": false,\n \"signature\": \"99b412bad31ee6959b473695ef09baa07b43e09ded03a4a254d60c25715f6059\"\n },\n {\n \"id\": \"ERR-H-L864-ERROR-20260715-DUPLICATE-MACHINE-MARKER-COUNT\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 864,\n \"title\": \"ERROR_20260715_DUPLICATE_MACHINE_MARKER_COUNT\",\n \"summary\": \"- Symptom: error-register candidate construction stopped with RC1 before applying the candidate. - Root cause: the same marker text appeared in both a heading and a descriptive line, while the guard expected one substring occurrence. - Correction: headings and prose do not contain machine-token values; validation counts only complete exact token lines. ANTI_REGRESSION_TOKEN=DUPLICATE_MACHINE_TOKEN_IN_HEADING_AND_BODY - Required pattern: grep -Fxc against a complete machine line, never grep -Foc against an unrestricted substring. - Production impact: none. - Temporary artifacts: removed and verified. - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": true,\n \"signature\": \"356ff9228e7bb1e236f72584597d788a4dc65367b0a1e4af38209a9b24be13ac\"\n },\n {\n \"id\": \"ERR-H-L875-ERROR-20260715-OUTER-WRAPPER-SHELL-SYNTAX\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 875,\n \"title\": \"ERROR_20260715_OUTER_WRAPPER_SHELL_SYNTAX\",\n \"summary\": \"- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis. - Root cause: the outer wrapper contained an unbalanced compound shell construct before the closing subshell delimiter. - Impact: parsing failed before the start marker, guards and homelab-admin invocation; no task phase or infrastructure action executed. - Correction: use a linear wrapper with explicitly closed if/case blocks and avoid nested brace groups in conditional expressions. ANTI_REGRESSION_TOKEN=OUTER_WRAPPER_UNBALANCED_SUBSHELL_SYNTAX - Anti-regression: after any parser-level failure, treat the attempted phase as not executed and verify package and run-state before retrying. - Production impact: none. - Task package impact: none.\",\n \"rule_like\": true,\n \"signature\": \"208c08a253c47c1ad448f2282eacc0af231ebeb3443f3f888206b3d7926d9eef\"\n },\n {\n \"id\": \"ERR-H-L888-ERROR-20260715-INLINE-REMOTE-SHELL-IN-APPLY-PHASE\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 888,\n \"title\": \"ERROR_20260715_INLINE_REMOTE_SHELL_IN_APPLY_PHASE\",\n \"summary\": \"- Symptom: homelab-admin lint rejected Stage4H task v6 with ERR-002-INLINE-REMOTE-SHELL. - Root cause: apply.sh invoked bash -s through an inline quoted SSH program and streamed the remote script through stdin. - Correction: copy the declared remote script to an isolated fixed path on VM180 and invoke that file directly with positional arguments. ANTI_REGRESSION_TOKEN=INLINE_REMOTE_SHELL_REPLACED_BY_STAGED_REMOTE_SCRIPT_PATH - Anti-regression: phase scripts may perform transport orchestration, but remote shell program bodies must exist as separately declared, syntax-checked remote script files. - Remote stdout and stderr must be preserved before evaluating the remote return code. - Task v6 status: REJECTED_BY_LINT; never executed and never mutated. - Production database impact: none.\",\n \"rule_like\": true,\n \"signature\": \"5c6fc9cd51fa9d557d4c8aee96a26b3d08bb3e86c6bd4813a7336a87c433b8bc\"\n },\n {\n \"id\": \"ERR-H-L902-ERROR-20260715-OPTIONAL-RUN-DIRECTORY-FIND-UNDER-PIPEFAIL\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 902,\n \"title\": \"ERROR_20260715_OPTIONAL_RUN_DIRECTORY_FIND_UNDER_PIPEFAIL\",\n \"summary\": \"- Symptom: task-v7 builder stopped while counting formal runs for rejected task v6. - Root cause: find was executed against an optional directory that did not exist; under pipefail the assignment returned RC1. - Correction: test directory existence first and assign zero without invoking find when it is absent. ANTI_REGRESSION_TOKEN=MISSING_OPTIONAL_RUN_DIRECTORY_TREATED_AS_ZERO - Anti-regression: optional paths must have an explicit existence branch before find under pipefail. - Production impact: none. - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0. - Registered at: 2026-07-15T04:44:06Z\",\n \"rule_like\": false,\n \"signature\": \"f5b1a0f37dcba3d28adda172281f4f4b53abc87f36ec3ed7fe35cf51845f0794\"\n },\n {\n \"id\": \"ERR-H-L912-ERROR-20260715-PG-CONSTRAINT-CONTYPE-CHAR-CONCAT\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 912,\n \"title\": \"ERROR_20260715_PG_CONSTRAINT_CONTYPE_CHAR_CONCAT\",\n \"summary\": \"- Symptom: isolated Stage4H acceptance failed during schema baseline capture. - Root cause: pg_constraint.contype uses the internal PostgreSQL char type and was concatenated without an explicit text cast. - Correction: use contype::text or CAST(contype AS text). ANTI_REGRESSION_TOKEN=PG_CATALOG_CHAR_CONCAT_REQUIRES_EXPLICIT_TEXT_CAST - Machine rule: ERR-014-PG-CATALOG-CHAR-CONCAT. - Negative self-test: uncast expression rejected with RC64. - Positive self-test: explicit text cast accepted with RC0. - Task v7 mutated: no.\",\n \"rule_like\": false,\n \"signature\": \"d269f342ff6fb119a139a91810419aa2a51ca9d24dcd0797ea65318c54ef10c7\"\n },\n {\n \"id\": \"ERR-H-L925-ERROR-20260715-LIVE-COLLECTOR-VERIFIER-WRONG-PATH\",\n \"kind\": \"heading\",\n \"level\": 2,\n \"source_line\": 925,\n \"title\": \"ERROR_20260715_LIVE_COLLECTOR_VERIFIER_WRONG_PATH\",\n \"summary\": \"- Symptom: verified linter installation was rolled back after the final collector integrity guard returned UNKNOWN. - Root cause: verifier queried a nonexistent path instead of the canonical live collector path declared by the task acceptance script. - Canonical path: /opt/cluster-admin-incident-engine/collector.py. - Incorrect path: /opt/homelab-cluster-admin/cluster-admin-collector.py. ANTI_REGRESSION_TOKEN=LIVE_COLLECTOR_VERIFIER_PATH_MUST_MATCH_TASK_CANONICAL_PATH - Correction: derive and guard the live collector path from the immutable task contract before executing the hash query. - Production impact: none; the canonical collector hash remained unchanged. - Status: CLOSED; UNRESOLVED_TAIL_COUNT=0.\",\n \"rule_like\": false,\n \"signature\": \"91e0cfb58ea5cec63e264f63f6ee65c13efe04ab4d4ab1f70a1d1f285f7b8ad2\"\n }\n ],\n \"rules\": [\n {\n \"id\": \"RULE-L3\",\n \"source_line\": 3,\n \"text\": \"Назначение: перед каждой следующей командой сверяться с этим файлом.\",\n \"signature\": \"910453eea5d379e1b055671660d8278df9d4565c580ecb6a036cfc1af83067f0\"\n },\n {\n \"id\": \"RULE-L11\",\n \"source_line\": 11,\n \"text\": \"## Жёсткие правила перед каждой командой\",\n \"signature\": \"3c373d80054bd99fe70a2f6e16e772fdaa67a63def211411b41f18d3123f91d0\"\n },\n {\n \"id\": \"RULE-L18\",\n \"source_line\": 18,\n \"text\": \"CHECK-7: если команда длиннее 8 строк, её нельзя давать в интерактивный shell.\",\n \"signature\": \"a9fa3d2bb79aa917672955c9adfcb227c45317ed178acb1a55b0c78505d7f10d\"\n },\n {\n \"id\": \"RULE-L36\",\n \"source_line\": 36,\n \"text\": \"Если rsync/scp упал, нельзя проверять latest offhost-файл без сверки имени.\",\n \"signature\": \"eabdf9a86e3c2ea3141f16847eb903bd516e6f12129d5e45df4f95c485794a2a\"\n },\n {\n \"id\": \"RULE-L46\",\n \"source_line\": 46,\n \"text\": \"Нельзя писать sqlite SQL вида j.type in ('object','array') внутри ssh '...'.\",\n \"signature\": \"668eb11f0a93aadac3d70e8792e586314755975dc4ac505ac13169cf55af6244\"\n },\n {\n \"id\": \"RULE-L55\",\n \"source_line\": 55,\n \"text\": \"Перед каждой командой явно понимать: где выполняется локальная часть, где remote-часть, каким пользователем.\",\n \"signature\": \"c8948237e08243f20cafec02cd56c65a19073cbef147809f7d16a7a1b027bfde\"\n },\n {\n \"id\": \"RULE-L59\",\n \"source_line\": 59,\n \"text\": \"Запрещено использовать в remote sqlite SQL фрагменты вида '$.key' или '.key'.\",\n \"signature\": \"10d38e69e3690c6892ab2aa8ddef391d5a3bdb71c9c1d80b23ebe30661d92071\"\n },\n {\n \"id\": \"RULE-L61\",\n \"source_line\": 61,\n \"text\": \"Команду с ошибкой char(36)||.dns_provider считать битой и не использовать.\",\n \"signature\": \"0ea37a4f0c68fcbd9d07b6ce4cc184b4152c5c247b1f31a1a70efa8cebeaacb9\"\n },\n {\n \"id\": \"RULE-L63\",\n \"source_line\": 63,\n \"text\": \"16. Основополагающее правило: перед каждой командой ассистент обязан явно написать: Сверка с файлом ошибок: CHECK пройден.\",\n \"signature\": \"ddd85ef9ebb1cbc0cb2d0cda3c70a5f4d23814cb7146f72672f99e2cbc8f82a0\"\n },\n {\n \"id\": \"RULE-L65\",\n \"source_line\": 65,\n \"text\": \"Нельзя выдавать команды без предварительной сверки с этим файлом ошибок.\",\n \"signature\": \"bd94bb5f40d6442f833a3ec028dfaf34c02e6d69701a7397257f9857202c2c5b\"\n },\n {\n \"id\": \"RULE-L66\",\n \"source_line\": 66,\n \"text\": \"Нельзя продолжать после собственной ошибки без записи ошибки в этот файл.\",\n \"signature\": \"f03bf4ecbe116c3281dd16b6ed1f80b16dc7a9ecd410a08244b5c41677fe5581\"\n },\n {\n \"id\": \"RULE-L68\",\n \"source_line\": 68,\n \"text\": \"19. Уточнение формата: не писать отдельную строку сверки перед каждой командой в пакете.\",\n \"signature\": \"4087c4bd5e52ca8c6f65e39b077c4bda9ae4a43aafac5857d5f33cc3f25683f0\"\n },\n {\n \"id\": \"RULE-L84\",\n \"source_line\": 84,\n \"text\": \"Не использовать больше сгенерированные большие base64 PHP для SMTP/XenForo.\",\n \"signature\": \"4db02d79ff907fa767992151c4a00ccff168749fa69417a6bcbeb6108656f2c0\"\n },\n {\n \"id\": \"RULE-L93\",\n \"source_line\": 93,\n \"text\": \"Нельзя использовать { echo RULE_CHECK_FAIL; exit 1; } в командах, выполняемых прямо в интерактивном root@pve01 shell.\",\n \"signature\": \"074e8d6400e50fd4a166ae4f67cf2da00d64051999145e17bbdaa432470ba64f\"\n },\n {\n \"id\": \"RULE-L105\",\n \"source_line\": 105,\n \"text\": \"Не использовать -crlf, если команды уже отправляются с явным \\\\r\\\\n.\",\n \"signature\": \"d8ee091ee5bb14359082cbc1e1b371d8128ddcff586b52516fed69b743f29cdf\"\n },\n {\n \"id\": \"RULE-L110\",\n \"source_line\": 110,\n \"text\": \"Не использовать heredoc Python внутри вложенного ssh; для dashboard verification использовать cat/grep JSON или простые команды.\",\n \"signature\": \"006d36b18ca6c52314d1e1d138393caf26b0d0e47533bf59ccba30210d12cbd7\"\n },\n {\n \"id\": \"RULE-L123\",\n \"source_line\": 123,\n \"text\": \"Нельзя считать monitoring слой runtime-closed только по secret scan/reference proof; нужно отдельно проверить Prometheus container/API и обновить справочник по факту.\",\n \"signature\": \"6744f09160604e98d226f32ef123dd61ee08278f0321c63edd729728300115e7\"\n },\n {\n \"id\": \"RULE-L125\",\n \"source_line\": 125,\n \"text\": \"31. Строгое правило: перед каждой командой сверяться с файлом ошибок и справочником.\",\n \"signature\": \"29acbf555ad438e9e4125ecb8c53e0ce38f6bce98ed3716e639afbb4c0eff3d7\"\n },\n {\n \"id\": \"RULE-L128\",\n \"source_line\": 128,\n \"text\": \"Если сверка не пройдена, команда не должна выполнять основное действие. Не использовать exit 1, чтобы не закрывать SSH-сессию.\",\n \"signature\": \"f20bbb59669123830e2c65160a0b67bf287cdcf82e94b17992919942fd715698\"\n },\n {\n \"id\": \"RULE-L132\",\n \"source_line\": 132,\n \"text\": \"Файл 153_PROMETHEUS_TARGETS_SETTLED_PROOF.txt нельзя считать валидным settled proof.\",\n \"signature\": \"8d28bb1cb84636890872e04ff869c76a8690936fed3b82af881bf1f2f83961e0\"\n },\n {\n \"id\": \"RULE-L133\",\n \"source_line\": 133,\n \"text\": \"Для таких проверок не использовать вложенный Python; использовать curl + grep/sed/awk либо отдельный локальный файл-скрипт.\",\n \"signature\": \"0d986ea7ea4c0cb4231812d5cf50fee6da9bdb43df2e61e2958c4536a56003b3\"\n },\n {\n \"id\": \"RULE-L176\",\n \"source_line\": 176,\n \"text\": \"- Do not split work into many small commands when the operations are low-risk, read-only, or naturally part of one evidence block.\",\n \"signature\": \"4ab34705facd281e3ea1b0cd209248f3b4087f623cf38f8b8622cbf511e67dea\"\n },\n {\n \"id\": \"RULE-L182\",\n \"source_line\": 182,\n \"text\": \"- Rule: avoid large opaque base64 script payloads for homelab changes; use transparent bounded shell/perl/python commands instead.\",\n \"signature\": \"3b1b7b83b7ed141a8dbf54ea052b6a3c6fb5701e0bc056939fa523727f996097\"\n },\n {\n \"id\": \"RULE-L185\",\n \"source_line\": 185,\n \"text\": \"- Rule: do not use large opaque payloads, generated base64 scripts, or hidden multi-line script blobs for homelab changes.\",\n \"signature\": \"386eaacb52062f19dc9f3fb9581a03ee50d5fbf08e19daba33f7271778b9b8e3\"\n },\n {\n \"id\": \"RULE-L188\",\n \"source_line\": 188,\n \"text\": \"- Any Traceback, UnicodeDecodeError, SyntaxError, Permission denied, command-not-found, nonzero critical RC, or failed write marker must force REVIEW or FAILED, never OK.\",\n \"signature\": \"540af43a414bcfd4a26cf5304ca785fbd5a127d99423647298862c964c3f493d\"\n },\n {\n \"id\": \"RULE-L196\",\n \"source_line\": 196,\n \"text\": \"- External informational widgets such as weather/Open-Meteo must not block the service launcher portal.\",\n \"signature\": \"74d434993ef9b2957b7beb9d50b3b35a30b84407a845454229c05e548bc91096\"\n },\n {\n \"id\": \"RULE-L201\",\n \"source_line\": 201,\n \"text\": \"- Rule: do not use deeply nested one-liners for config edits; first run short state checks, then apply small transparent edits.\",\n \"signature\": \"20fada2b3e3a676500b865a15816c133eb784aa10f4c6d7dab66220e573a8615\"\n },\n {\n \"id\": \"RULE-L206\",\n \"source_line\": 206,\n \"text\": \"- Matches in /logs/ or *.bak* are historical evidence only and must not make cleanup REVIEW unless the active config still contains the target.\",\n \"signature\": \"62de758aaf9d70c35a2f1fe076f80a8308e46d3b13d23c38665ab9f8ccc1cba4\"\n },\n {\n \"id\": \"RULE-L209\",\n \"source_line\": 209,\n \"text\": \"- Rule: Homepage link validation must not treat HTTP 200 as success if the body is the NPMPlus default “Congratulations” / host-not-set-up page.\",\n \"signature\": \"04b9d2a779820f3461cbb102ac47f6874198613575464488f460a9d287a089af\"\n },\n {\n \"id\": \"RULE-L217\",\n \"source_line\": 217,\n \"text\": \"- Rule: do not apply Netcraze config through stdin/multiline SSH; prove CLI input mode first.\",\n \"signature\": \"308cfe5fb5cfe99cb8c114de48913256103645e6a9e81ae9bc7ff446a484de9a\"\n },\n {\n \"id\": \"RULE-L228\",\n \"source_line\": 228,\n \"text\": \"- Rule: do not attempt ACL/config changes as routerbackup; use admin/operator router access or router Web UI for write changes.\",\n \"signature\": \"ce0e7d2dc85078db0180beacc1df838f3fe02f41db5de2d31573fe08f54a0f29\"\n },\n {\n \"id\": \"RULE-L245\",\n \"source_line\": 245,\n \"text\": \"- Rule: do not rely on custom Node lookup callback until separately tested; prefer curl --resolve or simple direct Node HTTP checks.\",\n \"signature\": \"25ad4cb5f7058222a9b5f59e98067f1db67a272e35cdd8292ca98eb050b3a0dc\"\n },\n {\n \"id\": \"RULE-L252\",\n \"source_line\": 252,\n \"text\": \"- Rule: do not keep changing ACL for this symptom; ACL is already correct. Fix service access scope or use a dedicated monitor endpoint.\",\n \"signature\": \"43f30d594ffb8d392ebc4b7defd82321e0f844a8b789a9b1d148a13df593d18c\"\n },\n {\n \"id\": \"RULE-L259\",\n \"source_line\": 259,\n \"text\": \"- Rule: do not use py_compile against root-owned system paths from an unprivileged user.\",\n \"signature\": \"ccf2aa95504d6021f9abcd8fecd6409a7a5efc61e8123fbbe7e03f539e979f89\"\n },\n {\n \"id\": \"RULE-L264\",\n \"source_line\": 264,\n \"text\": \"- Impact: do not use [PRIVATE_IP] directly as Homepage siteMonitor from edge-vm.\",\n \"signature\": \"8dfd3ac2d710c3ec6c9a2ec45f12611f948b98b8fe89d4d24b509a8d96082073\"\n },\n {\n \"id\": \"RULE-L279\",\n \"source_line\": 279,\n \"text\": \"- Rule: validate generated Python syntax locally before scp/remote execution; avoid f-strings in generated maintenance scripts.\",\n \"signature\": \"d7c5d3ed72ee048311445f823a903a5ecb6d22b03afec741f277aca10cd77858\"\n },\n {\n \"id\": \"RULE-L284\",\n \"source_line\": 284,\n \"text\": \"- Rule: do not proceed with OS baseline until SSH failure is diagnosed; likely old known_hosts key or cloud-init/root-key issue.\",\n \"signature\": \"9831f60cb061cc933e8c6688faad3f6b352d37d6237a03eefb5917a8facee420\"\n },\n {\n \"id\": \"RULE-L296\",\n \"source_line\": 296,\n \"text\": \"- Rule: avoid nested $(...) in VM SSH proofs; use literal remote commands and clean proof.\",\n \"signature\": \"65c4ac8d2adf3259f26e14c7d971b1ac68082da836bc807214901e58db59fdfa\"\n },\n {\n \"id\": \"RULE-L333\",\n \"source_line\": 333,\n \"text\": \"- Impact: do not trust that SQLite inspection attempt.\",\n \"signature\": \"23045ac8610d67d57dc1a7eb430578d6468f19d1acea9c7584f2a6f471393418\"\n },\n {\n \"id\": \"RULE-L334\",\n \"source_line\": 334,\n \"text\": \"- Rule: avoid nested SSH heredoc/Python for this task; use simpler commands, generated nginx configs, or uploaded/local script files.\",\n \"signature\": \"f4b937e830b202a02596fd60dbc1f2213645be3c5a70d929c8ff536f4adceb2c\"\n },\n {\n \"id\": \"RULE-L340\",\n \"source_line\": 340,\n \"text\": \"- Rule: read NPMplus API login values from docker inspect env internally, never print them.\",\n \"signature\": \"520fdfde2e139662a8738bb691cfe1929b9b9023a4dcc274e88a2b32ef1bfc5b\"\n },\n {\n \"id\": \"RULE-L351\",\n \"source_line\": 351,\n \"text\": \"- Impact: do not use NPMplus API for this publish path.\",\n \"signature\": \"dfce8b1c71b656add085d5c068f621a441177281533d880c8dc230b17b0e22dd\"\n },\n {\n \"id\": \"RULE-L357\",\n \"source_line\": 357,\n \"text\": \"- Impact: certbot ran without token and all certificate attempts failed; DNS cutover must not proceed from proof 535.\",\n \"signature\": \"a0e59d5dfd804bc63152339e3d2a3b0b288578d976b2039231a6272cbd9d002b\"\n },\n {\n \"id\": \"RULE-L406\",\n \"source_line\": 406,\n \"text\": \"- Rule: never source SMTP secret files containing arbitrary passwords; store password base64 and use msmtp passwordeval helper accessible to www-data.\",\n \"signature\": \"429fad764756578d3ed22029b6c2d6af0912b9b28d4a2263969cc53a090ba938\"\n },\n {\n \"id\": \"RULE-L419\",\n \"source_line\": 419,\n \"text\": \"- Rule: never store arbitrary passwords as shell-sourceable KEY=value files; use non-printing secret storage only.\",\n \"signature\": \"d04ca47a86fba60c038b212f585230b1bbb335493d583def83cbc3993bd7d88f\"\n },\n {\n \"id\": \"RULE-L424\",\n \"source_line\": 424,\n \"text\": \"- Rule: do not change DNS, firewall, proxy, mail, NetBird, or Mailcow configs before read-only triage proof.\",\n \"signature\": \"cdb23b83af44d2282df9c2a08be575953d61f280cc9abd53a4cc43b3e653cd1c\"\n },\n {\n \"id\": \"RULE-L430\",\n \"source_line\": 430,\n \"text\": \"- Rule: do not test SMTP auth again until a fresh rotated mailbox/app password is available.\",\n \"signature\": \"6694ecf8bef00c902be47d179510d8d4aadc93961c0164598e99350a41982f73\"\n },\n {\n \"id\": \"RULE-L436\",\n \"source_line\": 436,\n \"text\": \"- Impact: old timer must not be treated as valid current backup for all five forums.\",\n \"signature\": \"30e78abd678ecee3a63ddbd2d71e7caa390229722dab892fbfae0c891b725f13\"\n },\n {\n \"id\": \"RULE-L463\",\n \"source_line\": 463,\n \"text\": \"- Never print or package secrets.\",\n \"signature\": \"a0014dc66b74aee1b458771d46c51c80e582ee1bb69a1316aa6b63c91a3a40b2\"\n },\n {\n \"id\": \"RULE-L468\",\n \"source_line\": 468,\n \"text\": \"- Impact: proof 623 is invalid and must not be used to judge mail delivery.\",\n \"signature\": \"cdbba5ccdd73b604f4f3970ae1d69868a9d409f74ead6be3d60c031f26afa3f5\"\n },\n {\n \"id\": \"RULE-L497\",\n \"source_line\": 497,\n \"text\": \"- Do not print secrets.\",\n \"signature\": \"59a6b5bdc55ccb782cc5fa6d93fdf3d734c32d8250a2c21a57c6c7d4f6f3cd24\"\n },\n {\n \"id\": \"RULE-L498\",\n \"source_line\": 498,\n \"text\": \"- Do not download or upload:\",\n \"signature\": \"3de6b008bc8038faf7099e21fb98ec4280a409c05d3ae3453ec80bcd81e62240\"\n },\n {\n \"id\": \"RULE-L514\",\n \"source_line\": 514,\n \"text\": \"- Rule: do not use Cloudflare DNS-01 for these parked domains unless a token with explicit zone access is installed; because DNS already points to edge, prefer a copied-script HTTP-01 flow with temporary ACME challenge routes and content-specific validation.\",\n \"signature\": \"92d2f5a1cd4f344851af8d7a716dc24d8bc6a7738a47f6aa05013676f2200d61\"\n },\n {\n \"id\": \"RULE-L533\",\n \"source_line\": 533,\n \"text\": \"- Rule: do not run another apply stage until active NPMplus nginx route selection for newfi.ru is diagnosed with nginx -T and Host-header probes.\",\n \"signature\": \"039b358ea5679e61f5d7d8806ee36af893c18627bf8d26c434e902fb4c0e217f\"\n },\n {\n \"id\": \"RULE-L538\",\n \"source_line\": 538,\n \"text\": \"- Impact: HTTP-01 cannot work through the default server. A dedicated temporary ACME server block must be proven before certbot; do not run another certbot apply without a successful ACME-path probe for all six hostnames.\",\n \"signature\": \"2f1d3fca5be936885c2fff63166f1f43925dd8097efc48cfd46b54ff135f235a\"\n },\n {\n \"id\": \"RULE-L544\",\n \"source_line\": 544,\n \"text\": \"- Impact: do not continue certbot/apply until exact newfi.ru route behavior is isolated.\",\n \"signature\": \"8710aaa42a28a4c0a5598837af21305c7f4f17db34832d9e09cde58db0cf5a4b\"\n },\n {\n \"id\": \"RULE-L551\",\n \"source_line\": 551,\n \"text\": \"- Rule: do not use one combined parked-domain server_name block for these domains; use exact host blocks and content-specific validation.\",\n \"signature\": \"84a57025900a3fc4d1ba84dbcf54cf84b30875b82060e1e0dc66f9ccdf3da5ca\"\n },\n {\n \"id\": \"RULE-L558\",\n \"source_line\": 558,\n \"text\": \"- Rule: do not retry parked-domain HTTP-01 with low id 700-705. First prove fixed high-id exact blocks, then apply with those exact ids.\",\n \"signature\": \"527144862f28a73814ccc2c95fc4a380202a207f194962f9a7fb0a8d34833c2f\"\n },\n {\n \"id\": \"RULE-L564\",\n \"source_line\": 564,\n \"text\": \"- Rule: do not continue HTTP-01 for these parked domains; use DNS-01 with a dedicated Cloudflare token that has Zone Read and DNS Edit for the three zones.\",\n \"signature\": \"0c171db0e19ad3ff56b66bc5623f05867010cd7b2b87f98ef9cbe4d08a5c84f1\"\n },\n {\n \"id\": \"RULE-L568\",\n \"source_line\": 568,\n \"text\": \"- Impact: do not rerun Stage15 as-is.\",\n \"signature\": \"23033da11bdfd30e8d3405fa448f9e0a2516c09a778b90693eb8c01304e0b534\"\n },\n {\n \"id\": \"RULE-L569\",\n \"source_line\": 569,\n \"text\": \"- Rule: for parked domains finalization, use existing DNS-01 certificates and validate from pve01 public HTTPS with content marker; do not use the failed local --resolve TLS check as the closure gate.\",\n \"signature\": \"a6fd815206627d21af51c6ca97a5b2a03681011e1ad80eb0e6b29044cbce0323\"\n },\n {\n \"id\": \"RULE-L573\",\n \"source_line\": 573,\n \"text\": \"- Rule: Stage18 must not change DNS records or NPMplus route files. It may update placeholder HTML, run existing forum renewal script once for runtime proof, and run certbot dry-run for parked certificates using existing DNS-01 hooks.\",\n \"signature\": \"09d9b1e72b0bb7016128313662cddfa540a7a7908e768f8d4d22e5efd62168a0\"\n },\n {\n \"id\": \"RULE-L577\",\n \"source_line\": 577,\n \"text\": \"- Rule: only root gram1.ru and www.gram1.ru may be routed. Existing gram1.ru subdomain NPMplus routes must not be changed.\",\n \"signature\": \"dab2f98b8be402bac17f88d4e39ecfea8bdd5726097272cff11f7558f55eed0e\"\n },\n {\n \"id\": \"RULE-L578\",\n \"source_line\": 578,\n \"text\": \"- Rule: use DNS-01 with a dedicated gram1 Cloudflare token; do not use HTTP-01.\",\n \"signature\": \"40b057e3fd9659583584a2e3d4167a256076e134c4fbaa3678fcfc2503e55d2d\"\n },\n {\n \"id\": \"RULE-L583\",\n \"source_line\": 583,\n \"text\": \"- Decision: because pve01 has no SSH key access to the external VPS, do not modify the VPS. Instead move only pvepro.ru and www.pvepro.ru A records to edge and serve an edge landing page.\",\n \"signature\": \"17a7cd4c3204ffc9a003a8a6cef3e9be96599674d54f0f1a217eb1f3f0dad811\"\n },\n {\n \"id\": \"RULE-L584\",\n \"source_line\": 584,\n \"text\": \"- Safety rule: do not change mail.pvepro.ru, nb.pvepro.ru, MX, SPF, DKIM, DMARC or forum SMTP configuration.\",\n \"signature\": \"20b36bae3948a0b947ad12ca12a28cf8041c2ee4dd71dd71479c523fbeeaab0f\"\n },\n {\n \"id\": \"RULE-L585\",\n \"source_line\": 585,\n \"text\": \"- Use a dedicated Cloudflare token for pvepro.ru. Do not print token values.\",\n \"signature\": \"028d50fb2fef9139bce5a6a16069b7cdd69ab88719f640f0f0858034931d3d9b\"\n },\n {\n \"id\": \"RULE-L594\",\n \"source_line\": 594,\n \"text\": \"- Mistakes recorded: do not scp scripts/* after local py_compile because __pycache__ may be present; do not continue after critical scp failure in future apply stages; do not classify PVEPro landing as NetBird just because it contains a link text; use Cloudflare authoritative A records and --resolve validation when DNS propagation may lag.\",\n \"signature\": \"25354a6b9b469385d2300d5d321f8415312bb2392f2e96b37250717afce13b34\"\n },\n {\n \"id\": \"RULE-L600\",\n \"source_line\": 600,\n \"text\": \"- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet.\",\n \"signature\": \"b936ae30ffb935118585b5a133d9c32643946a3ba5034b85f42bae000518d49a\"\n },\n {\n \"id\": \"RULE-L606\",\n \"source_line\": 606,\n \"text\": \"- Rule: do not expose router admin publicly and do not attempt certificate deployment to the router over the public internet.\",\n \"signature\": \"b936ae30ffb935118585b5a133d9c32643946a3ba5034b85f42bae000518d49a\"\n },\n {\n \"id\": \"RULE-L611\",\n \"source_line\": 611,\n \"text\": \"- Rule: do not print WireGuard private keys, PSK, or client config body in proof/chat.\",\n \"signature\": \"d97ed9abb2447ca6786e0f266db8fe55e6071f854727ed9feb01a333d4fdd68e\"\n },\n {\n \"id\": \"RULE-L613\",\n \"source_line\": 613,\n \"text\": \"- Rule: do not open router public admin or certificate deployment until WireGuard handshake is proven.\",\n \"signature\": \"c44dbc849473687dad5c7756ca31b9b1366190f1f974a009ab8a2b2a175887cc\"\n },\n {\n \"id\": \"RULE-L641\",\n \"source_line\": 641,\n \"text\": \"- Do not retry immediately.\",\n \"signature\": \"e1208fd6483129c1940ee58ac88c1b14996e9906f5854c93a893423378ef3f63\"\n },\n {\n \"id\": \"RULE-L662\",\n \"source_line\": 662,\n \"text\": \"- Do not treat e3qxxx.netbird.selfhosted / 100.100.125.70 and e3qxxx-183-106.netbird.selfhosted / 100.100.183.106 as VPS egress nodes; they are user mobile peers.\",\n \"signature\": \"d025d035ce8bfa83c9f7b3605f86af415bded0a45359ee22348ce089f0a4de9f\"\n },\n {\n \"id\": \"RULE-L669\",\n \"source_line\": 669,\n \"text\": \"- Do not delete or disable Homepage siteMonitor fields to hide red badges.\",\n \"signature\": \"b101de9840659abe0fdbc458eeb21988b1ca2ef12a7cdce01e42f57991281d98\"\n },\n {\n \"id\": \"RULE-L671\",\n \"source_line\": 671,\n \"text\": \"- Do not touch Cloudflare Homepage card when operator says it is green and opens correctly.\",\n \"signature\": \"e55eaa04a75332bfdad36576da5425a956fa6e84f1003a1c606e64b0701737a5\"\n },\n {\n \"id\": \"RULE-L678\",\n \"source_line\": 678,\n \"text\": \"- Do not touch Cloudflare when operator says it is green and opens correctly.\",\n \"signature\": \"df725b9f5c7198ce4c04f9c37c1a2705147a9b2547a8f6de89800c03c68ea37f\"\n },\n {\n \"id\": \"RULE-L684\",\n \"source_line\": 684,\n \"text\": \"- Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions.\",\n \"signature\": \"b31b7d7bef8b7f89787850a3c73ce500918e256710e16626efa0d983dbec9b31\"\n },\n {\n \"id\": \"RULE-L711\",\n \"source_line\": 711,\n \"text\": \"- Future preparation must use exact privileged paths or a verified shared group, never an unprivileged wildcard after restrictive chmod.\",\n \"signature\": \"ffadddc535d9f6ef63a2e5f96282815f86c2844d21a1302af925758271cf605f\"\n },\n {\n \"id\": \"RULE-L737\",\n \"source_line\": 737,\n \"text\": \"- The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults.\",\n \"signature\": \"edb305f9a22332f3efb7192d483e3f067a1c50b93b0152596c8bbdff8b58fcd3\"\n },\n {\n \"id\": \"RULE-L768\",\n \"source_line\": 768,\n \"text\": \"- The phase-wide forbidden-pattern scan also included validate_design.py itself, which necessarily contains the prohibited literals it verifies.\",\n \"signature\": \"c9daa84dfbc8edb3cc4046148bdcb08f590a765c069f15dccba110171dcfefee\"\n },\n {\n \"id\": \"RULE-L789\",\n \"source_line\": 789,\n \"text\": \"- The backup must not be rerun merely to satisfy the invalid health-file freshness assertion.\",\n \"signature\": \"85bc7d2eeb4c251bf6db4f1e1e9be31bd05c124e10a3c3d0c9b671f1c2aea449\"\n },\n {\n \"id\": \"RULE-L793\",\n \"source_line\": 793,\n \"text\": \"- Mandatory anti-regression: после ошибки запрещено продолжать другой scope до точной root cause, исправления, повторной проверки, очистки, proof и seal.\",\n \"signature\": \"3506ad8423c9306313c4995dffee63340e95a4e1b0fdab450b433570927d3602\"\n },\n {\n \"id\": \"RULE-L818\",\n \"source_line\": 818,\n \"text\": \"- Anti-regression: never combine sudo with `< protected-file`; pass the protected path as an argument to the privileged command.\",\n \"signature\": \"8b5571b92d02dbe05c3f8a9f7637776e8665b44ce49521d4fe408fa977c13e39\"\n },\n {\n \"id\": \"RULE-L825\",\n \"source_line\": 825,\n \"text\": \"- Correction: assert required and forbidden column counts before querying recent runs.\",\n \"signature\": \"598215d2c23dd83bd6faa1beedb94f32b94ac3fd7e0573ea754db0f680b415ab\"\n },\n {\n \"id\": \"RULE-L834\",\n \"source_line\": 834,\n \"text\": \"- Anti-regression: stdin-скрипт обязан валидировать $1 по разрешённому path pattern и затем подтверждать точные SHA256 и byte count; запрещено добавлять отдельный bash-token после remote command.\",\n \"signature\": \"333aa0cc07135a79e5044a85b773ad5a7f0e655fd0a0d6422ec994230ce8112a\"\n },\n {\n \"id\": \"RULE-L867\",\n \"source_line\": 867,\n \"text\": \"- Correction: headings and prose do not contain machine-token values; validation counts only complete exact token lines.\",\n \"signature\": \"d820b99656b281c428c56ed8698a3fe4295e435e708263040de0136d1a42a18d\"\n },\n {\n \"id\": \"RULE-L869\",\n \"source_line\": 869,\n \"text\": \"- Required pattern: grep -Fxc against a complete machine line, never grep -Foc against an unrestricted substring.\",\n \"signature\": \"bab78eea9eadb2614ae16292dec8fda0bead38c2223efd2c79b512d1ad965836\"\n },\n {\n \"id\": \"RULE-L879\",\n \"source_line\": 879,\n \"text\": \"- Correction: use a linear wrapper with explicitly closed if/case blocks and avoid nested brace groups in conditional expressions.\",\n \"signature\": \"e6627533fc6f44113d3e3fb415ee4807a46afce3393d2457ca87649f3e95a338\"\n },\n {\n \"id\": \"RULE-L895\",\n \"source_line\": 895,\n \"text\": \"- Task v6 status: REJECTED_BY_LINT; never executed and never mutated.\",\n \"signature\": \"eaa651690a1e2b17124007e55346be7ad52d4d9f6172746262bc0698c376396d\"\n }\n ],\n \"command_ledger\": {\n \"present\": true,\n \"entry_count\": 10,\n \"failed_entry_count\": 0,\n \"failed_command_hashes\": []\n },\n \"privacy\": {\n \"sanitized\": true,\n \"secrets_included\": false,\n \"private_addresses_included\": false\n }\n}\n" +} +DETAIL=SECTION_STRUCTURED_ERRORS_INDEX_END +DETAIL=SECTION_WORKFLOW_CONTRACT_BEGIN +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "WORKFLOW-CONTRACT-003", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "change", + "component": "assistant-workflow", + "started_at_utc": "2026-07-21T07:41:45Z", + "finished_at_utc": "2026-07-21T07:41:45Z", + "reference_register_checked": true, + "reference_sha256": "f6f7749fda946015116cdbbd754c7e7f20356b4a6cf4df330a1b0b8fdf59cbb2", + "error_register_checked": true, + "error_register_sha256": "ab8308549db90d235b591af0581580c3c7ba1b287b129cc6111c2eef0a0f9ecc", + "command_sha256": "d3ec6690bfb2ddbf543d9adbe2ca25d4296e193eb398776f170693ff84e91d39", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": true, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "30d935ecf9ca36e222dada4ade200a38d0066cce09656c79a60b1f25add68dc6", + "sanitized_output_sha256": "30d935ecf9ca36e222dada4ade200a38d0066cce09656c79a60b1f25add68dc6", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "WORKFLOW_CONTRACT_VERSION=3\nCONFIRM_BACKSLASH=yes\nCONFIRM_COMMA=yes\nCONFIRM_MEANING=COMMAND_EXECUTED_READ_EXACT_IMMUTABLE_RESULT\nMANUAL_TERMINAL_OUTPUT=no\nFILE_UPLOADS=no\nCOMMAND_DISPLAY=COLLAPSED_SPOILER\nCOMMANDS_PER_STEP=1\nCOMMAND_STYLE=SHORT_TRANSPARENT\nHEREDOC_FORBIDDEN=yes\nBASE64_COMMANDS_FORBIDDEN=yes\nLARGE_INLINE_COMMANDS_FORBIDDEN=yes\nRESULT_SOURCE=IMMUTABLE_HISTORY_EXACT_COMMAND_ID\nLATEST_JSON_AUTHORITATIVE=no\nBEFORE_COMMAND=LOAD_BOOTSTRAP_ERRORS_INDEX_PREVIOUS_RESULT\nAFTER_FAILURE=READ_PUBLISHED_FAILURE_BEFORE_NEXT_COMMAND\nNO_REPEAT_EXACT_FAILED_COMMAND=yes\nMISSING_RESULT_CHECK=EXACT_FILE_THEN_HISTORY_DIRECTORY_THEN_BRANCH_HEAD\nNEW_CHAT_INPUT=BOOTSTRAP_URL_ONLY\n" +} +DETAIL=SECTION_WORKFLOW_CONTRACT_END +DETAIL=SECTION_DOUBLE_CHECK_RULE_BEGIN +{"schema_version":1,"command_id":"DOUBLE-CHECK-RULE-001","status":"OK","strict":true,"rule":"two_independent_pre_command_self_checks","checks":["bootstrap_errors_previous_result_failed_hash","syntax_quoting_target_permissions_side_effects_rollback_success_criteria"]} +DETAIL=SECTION_DOUBLE_CHECK_RULE_END +DETAIL=SECTION_FRESH_AUTO_CONTEXT_BEGIN +CHAT_OUTPUT_BEGIN +COMMAND_ID=CONTEXT-AUTO-[PRIVATE_IP]652Z STATUS=OK RC=0 HOST=pve01 @@ -7,7 +3893,7 @@ COMPONENT=cluster-context REFERENCE_SHA[PRIVATE_IP]f8edc75fcf[PRIVATE_IP]c[PRIVATE_IP]cc112ceccb6cf[PRIVATE_IP]a4deb8e ERROR_REGISTER_SHA[PRIVATE_IP]b8e38eaa1acec915352fb[PRIVATE_IP]caa7ff6cd9d1ec0da[PRIVATE_IP] OUTPUT_BEGIN -GENERATED_AT_UTC=[PRIVATE_IP]T14:17:01Z +GENERATED_AT_UTC=[PRIVATE_IP]T14:16:52Z Cluster information ------------------- Name: homelab @@ -45,7 +3931,7 @@ Membership information 1 1 pve01 (local) 2 1 pve03 3 1 pve02 -[{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/110","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns1","netin":[PRIVATE_IP],"netout":641869,"node":"pve01","status":"running","template":0,"type":"lxc","uptime":3356,"vmid":110},{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/111","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns2","netin":[PRIVATE_IP],"netout":272832,"node":"pve02","status":"running","template":0,"type":"lxc","uptime":3188,"vmid":111},{"cpu":[PRIVATE_IP]224e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/112","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound1","netin":[PRIVATE_IP],"netout":780812,"node":"pve01","status":"running","template":0,"type":"lxc","uptime":3358,"vmid":112},{"cpu":[PRIVATE_IP]509e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/113","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound2","netin":[PRIVATE_IP],"netout":184907,"node":"pve02","status":"running","template":0,"type":"lxc","uptime":3191,"vmid":113},{"cpu":[PRIVATE_IP]63921,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"edge-vm","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":3296,"vmid":130},{"cpu":[PRIVATE_IP]57485,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/150","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"nextcloud","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":3353,"vmid":150},{"cpu":[PRIVATE_IP]575575,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/160","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"forum-prod","netin":[PRIVATE_IP],"netout":66408,"node":"pve02","status":"running","template":0,"type":"qemu","uptime":3185,"vmid":160},{"cpu":[PRIVATE_IP]4996,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/170","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"core-apps","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":3349,"vmid":170},{"cpu":[PRIVATE_IP]74928,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/171","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"monitoring","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":3346,"vmid":171},{"cpu":[PRIVATE_IP]029885,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/180","maxcpu":2,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"cluster-admin","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"qemu","uptime":3182,"vmid":180},{"cpu":[PRIVATE_IP]175948,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/190","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"homelab-ops-worker","netin":[PRIVATE_IP],"netout":194385,"node":"pve03","status":"running","template":0,"type":"qemu","uptime":3292,"vmid":190},{"cpu":0,"disk":0,"diskread":0,"diskwrite":0,"id":"qemu/9130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":0,"memhost":0,"name":"edge-cold-standby","netin":0,"netout":0,"node":"pve02","status":"stopped","template":0,"type":"qemu","uptime":0,"vmid":9130},{"cgroup-mode":2,"cpu":[PRIVATE_IP]45763,"disk":[PRIVATE_IP],"id":"node/pve03","level":"","maxcpu":8,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve03","status":"online","type":"node","uptime":3314},{"cgroup-mode":2,"cpu":[PRIVATE_IP]58711,"disk":[PRIVATE_IP],"id":"node/pve01","level":"","maxcpu":24,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve01","status":"online","type":"node","uptime":3386},{"cgroup-mode":2,"cpu":[PRIVATE_IP]14972,"disk":[PRIVATE_IP],"id":"node/pve02","level":"","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve02","status":"online","type":"node","uptime":3252},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve03/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve01/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve02/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"backup,iso,import,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve03/local","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"backup,iso,import,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve01/local","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"backup,iso,import,vztmpl","disk":[PRIVATE_IP],"id":"storage/pve02/local","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"id":"network/pve03/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve03","status":"ok","type":"network"},{"id":"network/pve01/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve01","status":"ok","type":"network"},{"id":"network/pve02/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve02","status":"ok","type":"network"}] +[{"cpu":[PRIVATE_IP]801e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/110","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns1","netin":[PRIVATE_IP],"netout":641185,"node":"pve01","status":"running","template":0,"type":"lxc","uptime":3345,"vmid":110},{"cpu":[PRIVATE_IP][PRIVATE_IP],"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/111","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"dns2","netin":[PRIVATE_IP],"netout":272209,"node":"pve02","status":"running","template":0,"type":"lxc","uptime":3178,"vmid":111},{"cpu":[PRIVATE_IP]801e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/112","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound1","netin":[PRIVATE_IP],"netout":780728,"node":"pve01","status":"running","template":0,"type":"lxc","uptime":3348,"vmid":112},{"cpu":[PRIVATE_IP]424e-05,"disk":[PRIVATE_IP],"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"lxc/113","maxcpu":1,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":0,"name":"unbound2","netin":[PRIVATE_IP],"netout":184449,"node":"pve02","status":"running","template":0,"type":"lxc","uptime":3181,"vmid":113},{"cpu":[PRIVATE_IP]20667,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"edge-vm","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve03","status":"running","template":0,"type":"qemu","uptime":3286,"vmid":130},{"cpu":[PRIVATE_IP]49185,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/150","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"nextcloud","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":3343,"vmid":150},{"cpu":[PRIVATE_IP]98795,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/160","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"forum-prod","netin":[PRIVATE_IP],"netout":65626,"node":"pve02","status":"running","template":0,"type":"qemu","uptime":3175,"vmid":160},{"cpu":[PRIVATE_IP]05248,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/170","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"core-apps","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":3339,"vmid":170},{"cpu":[PRIVATE_IP]91491,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/171","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"monitoring","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve01","status":"running","template":0,"type":"qemu","uptime":3336,"vmid":171},{"cpu":[PRIVATE_IP]31022,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/180","maxcpu":2,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"cluster-admin","netin":[PRIVATE_IP],"netout":[PRIVATE_IP],"node":"pve02","status":"running","template":0,"type":"qemu","uptime":3172,"vmid":180},{"cpu":[PRIVATE_IP]87303,"disk":0,"diskread":[PRIVATE_IP],"diskwrite":[PRIVATE_IP],"id":"qemu/190","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"memhost":[PRIVATE_IP],"name":"homelab-ops-worker","netin":[PRIVATE_IP],"netout":194110,"node":"pve03","status":"running","template":0,"type":"qemu","uptime":3282,"vmid":190},{"cpu":0,"disk":0,"diskread":0,"diskwrite":0,"id":"qemu/9130","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":0,"memhost":0,"name":"edge-cold-standby","netin":0,"netout":0,"node":"pve02","status":"stopped","template":0,"type":"qemu","uptime":0,"vmid":9130},{"cgroup-mode":2,"cpu":[PRIVATE_IP]24746,"disk":[PRIVATE_IP],"id":"node/pve02","level":"","maxcpu":4,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve02","status":"online","type":"node","uptime":3242},{"cgroup-mode":2,"cpu":[PRIVATE_IP]96147,"disk":[PRIVATE_IP],"id":"node/pve03","level":"","maxcpu":8,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve03","status":"online","type":"node","uptime":3303},{"cgroup-mode":2,"cpu":[PRIVATE_IP]41631,"disk":[PRIVATE_IP],"id":"node/pve01","level":"","maxcpu":24,"maxdisk":[PRIVATE_IP],"maxmem":[PRIVATE_IP],"mem":[PRIVATE_IP],"node":"pve01","status":"online","type":"node","uptime":3376},{"content":"iso,import,vztmpl,backup","disk":[PRIVATE_IP],"id":"storage/pve02/local","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"iso,import,vztmpl,backup","disk":[PRIVATE_IP],"id":"storage/pve03/local","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"iso,import,vztmpl,backup","disk":[PRIVATE_IP],"id":"storage/pve01/local","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"dir","shared":0,"status":"available","storage":"local","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve02/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve02","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve03/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve03","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"content":"rootdir,images","disk":[PRIVATE_IP],"id":"storage/pve01/local-lvm","maxdisk":[PRIVATE_IP],"node":"pve01","plugintype":"lvmthin","shared":0,"status":"available","storage":"local-lvm","type":"storage"},{"id":"network/pve02/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve02","status":"ok","type":"network"},{"id":"network/pve03/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve03","status":"ok","type":"network"},{"id":"network/pve01/zone/localnetwork","network":"localnetwork","network-type":"zone","node":"pve01","status":"ok","type":"network"}] Name Type Status Total (KiB) Used (KiB) Available (KiB) % local dir active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 41.13% local-lvm lvmthin active [PRIVATE_IP] [PRIVATE_IP] [PRIVATE_IP] 38.36% @@ -59,27 +3945,25 @@ Legend: LOAD → Reflects whether the unit definition was properly loaded. 2 loaded units listed. NEXT LEFT LAST PASSED UNIT ACTIVATES -Thu [PRIVATE_IP] 17:17:11 MSK 8s Thu [PRIVATE_IP] 17:16:07 MSK 55s ago homelab-router-watchdog.timer homelab-router-watchdog.service -Thu [PRIVATE_IP] 17:17:15 MSK 12s Thu [PRIVATE_IP] 17:12:15 MSK 4min 47s ago homelab-smartctl-textfile.timer homelab-smartctl-textfile.service -Thu [PRIVATE_IP] 17:17:19 MSK 17s Thu [PRIVATE_IP] 17:12:08 MSK 4min 54s ago homelab-health-metrics.timer homelab-health-metrics.service -Thu [PRIVATE_IP] 17:17:28 MSK 25s Thu [PRIVATE_IP] 17:16:28 MSK 34s ago homelab-cluster-admin-webpanel-sync.timer homelab-cluster-admin-webpanel-sync.service -Thu [PRIVATE_IP] 17:17:28 MSK 25s Thu [PRIVATE_IP] 17:16:28 MSK 34s ago homelab-private-vpn-hosts-health.timer homelab-private-vpn-hosts-health.service -Thu [PRIVATE_IP] 17:17:32 MSK 29s Thu [PRIVATE_IP] 17:12:08 MSK 4min 54s ago homelab-crypto-portfolio-chart-health.timer homelab-crypto-portfolio-chart-health.service -Thu [PRIVATE_IP] 17:18:28 MSK 1min 25s Thu [PRIVATE_IP] 17:13:28 MSK 3min 34s ago netbird-peers-health.timer netbird-peers-health.service -Thu [PRIVATE_IP] 17:18:56 MSK 1min 53s Thu [PRIVATE_IP] 17:16:53 MSK 9s ago homelab-cluster-admin-inventory-sync.timer homelab-cluster-admin-inventory-sync.service -Thu [PRIVATE_IP] 17:20:58 MSK 3min 55s Thu [PRIVATE_IP] 17:05:58 MSK 11min ago prometheus-node-exporter-apt.timer prometheus-node-exporter-apt.service -Thu [PRIVATE_IP] 17:20:58 MSK 3min 55s Thu [PRIVATE_IP] 17:05:58 MSK 11min ago prometheus-node-exporter-nvme.timer prometheus-node-exporter-nvme.service +Thu [PRIVATE_IP] 17:17:11 MSK 18s Thu [PRIVATE_IP] 17:16:07 MSK 45s ago homelab-router-watchdog.timer homelab-router-watchdog.service +Thu [PRIVATE_IP] 17:17:15 MSK 22s Thu [PRIVATE_IP] 17:12:15 MSK 4min 37s ago homelab-smartctl-textfile.timer homelab-smartctl-textfile.service +Thu [PRIVATE_IP] 17:17:19 MSK 26s Thu [PRIVATE_IP] 17:12:08 MSK 4min 44s ago homelab-health-metrics.timer homelab-health-metrics.service +Thu [PRIVATE_IP] 17:17:28 MSK 35s Thu [PRIVATE_IP] 17:16:28 MSK 24s ago homelab-cluster-admin-webpanel-sync.timer homelab-cluster-admin-webpanel-sync.service +Thu [PRIVATE_IP] 17:17:28 MSK 35s Thu [PRIVATE_IP] 17:16:28 MSK 24s ago homelab-private-vpn-hosts-health.timer homelab-private-vpn-hosts-health.service +Thu [PRIVATE_IP] 17:17:32 MSK 39s Thu [PRIVATE_IP] 17:12:08 MSK 4min 44s ago homelab-crypto-portfolio-chart-health.timer homelab-crypto-portfolio-chart-health.service +Thu [PRIVATE_IP] 17:18:28 MSK 1min 35s Thu [PRIVATE_IP] 17:13:28 MSK 3min 24s ago netbird-peers-health.timer netbird-peers-health.service +Thu [PRIVATE_IP] 17:20:58 MSK 4min 5s Thu [PRIVATE_IP] 17:05:58 MSK 10min ago prometheus-node-exporter-apt.timer prometheus-node-exporter-apt.service +Thu [PRIVATE_IP] 17:20:58 MSK 4min 5s Thu [PRIVATE_IP] 17:05:58 MSK 10min ago prometheus-node-exporter-nvme.timer prometheus-node-exporter-nvme.service Thu [PRIVATE_IP] 17:23:15 MSK 6min Thu [PRIVATE_IP] 17:08:15 MSK 8min ago homelab-disk-space-health.timer homelab-disk-space-health.service Thu [PRIVATE_IP] 17:23:15 MSK 6min Thu [PRIVATE_IP] 17:08:15 MSK 8min ago homelab-pve03-staging-capacity-health.timer homelab-pve03-staging-capacity-health.service -Thu [PRIVATE_IP] 17:26:04 MSK 9min Thu [PRIVATE_IP] 17:16:04 MSK 58s ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service -Thu [PRIVATE_IP] 17:30:00 MSK 12min Thu [PRIVATE_IP] 17:15:04 MSK 1min 58s ago homelab-incident-journal.timer homelab-incident-journal.service -Thu [PRIVATE_IP] 17:30:00 MSK 12min Thu [PRIVATE_IP] 17:15:04 MSK 1min 58s ago homelab-safe-autoheal.timer homelab-safe-autoheal.service -Thu [PRIVATE_IP] 17:30:14 MSK 13min Thu [PRIVATE_IP] 17:16:09 MSK 53s ago homelab-duty-admin-v2.timer homelab-duty-admin-v2.service -Thu [PRIVATE_IP] 17:30:30 MSK 13min Thu [PRIVATE_IP] 17:16:04 MSK 58s ago homelab-external-probe-vps-health.timer homelab-external-probe-vps-health.service -Thu [PRIVATE_IP] 17:30:42 MSK 13min Thu [PRIVATE_IP] 17:16:04 MSK 58s ago homelab-forum-snuffleupagus-health.timer homelab-forum-snuffleupagus-health.service -Thu [PRIVATE_IP] 17:32:05 MSK 15min Thu [PRIVATE_IP] 17:04:00 MSK 13min ago homelab-mkdocs-auto-refresh.timer homelab-mkdocs-auto-refresh.service +Thu [PRIVATE_IP] 17:26:04 MSK 9min Thu [PRIVATE_IP] 17:16:04 MSK 48s ago skladchik-reports-monitor-supervisor.timer skladchik-reports-monitor-supervisor.service +Thu [PRIVATE_IP] 17:30:00 MSK 13min Thu [PRIVATE_IP] 17:15:04 MSK 1min 48s ago homelab-incident-journal.timer homelab-incident-journal.service +Thu [PRIVATE_IP] 17:30:00 MSK 13min Thu [PRIVATE_IP] 17:15:04 MSK 1min 48s ago homelab-safe-autoheal.timer homelab-safe-autoheal.service +Thu [PRIVATE_IP] 17:30:14 MSK 13min Thu [PRIVATE_IP] 17:16:09 MSK 43s ago homelab-duty-admin-v2.timer homelab-duty-admin-v2.service +Thu [PRIVATE_IP] 17:30:30 MSK 13min Thu [PRIVATE_IP] 17:16:04 MSK 48s ago homelab-external-probe-vps-health.timer homelab-external-probe-vps-health.service +Thu [PRIVATE_IP] 17:32:05 MSK 15min Thu [PRIVATE_IP] 17:04:00 MSK 12min ago homelab-mkdocs-auto-refresh.timer homelab-mkdocs-auto-refresh.service Thu [PRIVATE_IP] 17:39:47 MSK 22min Thu [PRIVATE_IP] 17:08:37 MSK 8min ago homelab-reference-refresh.timer homelab-reference-refresh.service -Thu [PRIVATE_IP] 21:43:01 MSK 4h 25min Wed [PRIVATE_IP] 23:00:09 MSK - apt-daily.timer apt-daily.service +Thu [PRIVATE_IP] 21:43:01 MSK 4h 26min Wed [PRIVATE_IP] 23:00:09 MSK - apt-daily.timer apt-daily.service Thu [PRIVATE_IP] 22:23:37 MSK 5h 6min Thu [PRIVATE_IP] 16:23:37 MSK 53min ago homelab-vps-identity-audit.timer homelab-vps-identity-audit.service Fri [PRIVATE_IP] 00:00:00 MSK 6h Thu [PRIVATE_IP] 00:00:07 MSK - dpkg-db-backup.timer dpkg-db-backup.service Fri [PRIVATE_IP] 00:03:01 MSK 6h Thu [PRIVATE_IP] 00:03:53 MSK - homelab-evidence-catalog.timer homelab-evidence-catalog.service @@ -152,6 +4036,8 @@ Mon [PRIVATE_IP] 00:07:15 MSK 3 days Mon [PRIVATE_IP] 00:00:29 MSK Mon [PRIVATE_IP] 01:33:11 MSK 3 days Mon [PRIVATE_IP] 00:51:14 MSK - fstrim.timer fstrim.service Mon [PRIVATE_IP] 07:34:52 MSK 3 days Mon [PRIVATE_IP] 07:25:14 MSK - homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service Thu [PRIVATE_IP] 06:00:00 MSK 2 months 22 days Wed [PRIVATE_IP] 06:00:00 MSK - homelab-vm-full-restore-drill.timer homelab-vm-full-restore-drill.service +- - Thu [PRIVATE_IP] 17:16:53 MSK 17ms ago homelab-cluster-admin-inventory-sync.timer homelab-cluster-admin-inventory-sync.service +- - Thu [PRIVATE_IP] 17:16:04 MSK 48s ago homelab-forum-snuffleupagus-health.timer homelab-forum-snuffleupagus-health.service - - - - prometheus-node-exporter-ipmitool-sensor.timer prometheus-node-exporter-ipmitool-sensor.service - - - - prometheus-node-exporter-mellanox-hca-temp.timer prometheus-node-exporter-mellanox-hca-temp.service - - - - prometheus-node-exporter-smartmon.timer prometheus-node-exporter-smartmon.service @@ -199,3 +4085,10 @@ BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error nea Resolution: avoid nested quote layers; parse targets first and pass file paths as simple SSH arguments. OUTPUT_END CHAT_OUTPUT_END +DETAIL=SECTION_FRESH_AUTO_CONTEXT_END +DETAIL=FULL_CONTEXT_BUNDLE_END +DETAIL=CONTEXT_SOURCES_FULL_REFRESH=PASS +CHANGES_MADE=NO + +OUTPUT_END +CHAT_OUTPUT_END