CHAT_OUTPUT_BEGIN
COMMAND_ID=CRITICAL-FIRST-ACTION-102
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=a1e6caadaf21754e0be5b91308d72b83a99e63dd63ac42090ae387854cc58c71
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=3079d39893301d0f2225f12640da1f5cddeddd6666f812a8fd0abf78c3d5f998
SANITIZED_OUTPUT_SHA256=3079d39893301d0f2225f12640da1f5cddeddd6666f812a8fd0abf78c3d5f998
OUTPUT_BEGIN
=== VERIFIED_PREVIOUS_RESULT ===
COMMAND_ID=CRITICAL-TAILS-FINDINGS-101
IMMUTABLE_COMMIT=c8c9e3736cc5a1723427b06fcd8e8e2ee48903be
cut: '"': No such file or directory
JSON_SHA256=
cut: '"': No such file or directory
TXT_SHA256=
PREVIOUS_COMMAND_IN_ERROR_REGISTER=no
=== FINDING_COUNTS ===
P0_MATCHES=45
P1_MATCHES=87
STORAGE_RISK_MATCHES=2
RECENT_FAILED_COMMAND_LINES=0
BOOTSTRAP_SECTION_LINES=0
=== ACTIONABLE_P0 ===
=== ACTIONABLE_P0 ===
86:398:- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained.
155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.
203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.
486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
546:● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state
548:● homelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03
1102:/var/lib/homelab-health/backup-matrix-cluster-final-seal.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
1104:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-production-closure.txt:1:STATUS=OK TS=2026-07-13T08:12:14Z TYPE=crypto-portfolio-ui-center-alert-currency-production-closure BAD=0 PRODUCTION_CHANGED=YES_UI_ONLY INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 POSTCHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION_ROLLBACK_COUNT=3 RETENTION_OPTIONAL_OTHER_CANDIDATES=1 RETENTION_REMOVED_ROLLBACK_COUNT=0 DANGLING_IMAGES_REMAINING=2 UNREFERENCED_DANGLING_IMAGES=0 DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE_CLEAN=YES PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES READY_FOR_UI_FINAL_SEAL=YES
1106:/var/lib/homelab-health/cluster-admin-stage3-postapply-backup.txt:1:STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z
1109:/var/lib/homelab-health/pve01-core-monitoring-app-restorecheck.txt:1:STATUS=OK TS=2026-07-08T15:57:27Z TYPE=pve01-core-monitoring-app-restorecheck BAD=0 MODE=APP_LEVEL_RESTORE_CHECKS_TWO_CLOUDS FULL_VM_RESTORE_AVAILABLE=NO TOTAL_APPS=30 OK_APPS=30 FAIL_LIST=none REPORT=/var/lib/homelab-node-loss/pve01-core-monitoring-app-level/app-restorecheck-latest.md
1118:/var/lib/homelab-health/hapusya-site-final-seal.txt:1:STATUS=OK TS=2026-07-13T12:33:08Z TYPE=hapusya-site-final-seal BAD=0 APP=hapusya-site URL=https://hapusya.ru/ PRESEAL_DESIRED_STATE_HEAD=69966eb83fc065141df3c46df0d6c0f282275046 ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb APPLY_MANIFEST_SHA256=f44992e3e0cc443b372779cb35d9e899d904a9960a9de03c14559886fdb1fc75 PRODUCTION_BROWSER_MANIFEST_SHA256=9f3cdf3063f8724d1d1c1ad619f47d6f98d434c3422a74903585146858a8dfac BROWSER_SMOKE=OK BROWSER_RUNS=5 BROWSER_ERRORS=0 DESKTOP_VIDEO=OK MOBILE_VIDEO=OK STATIC_FALLBACK=OK BACKUP_JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 TWO_CLOUDS=OK RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK DESIRED_STATE_SYNC=OK GENERATOR_SHA256=507bbbc2691f1f08cb1dd3660ed76cf13a8c36c23833d5fdec40cf6e07486cb8 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 POLICY_SHA256=4bf996ff451ae3ec56eb2fcd42842606cd97295b4d5141aedb135c956395e4b5 EVIDENCE_ROOT=/var/lib/homelab-change-sets/hapusya-site-final-seal/20260713T123308Z EVIDENCE_MANIFEST_SHA256=7eb993a17ac18a1f7c9ae0b52d745d3778833d15626ed34c82496f74916eb26a PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY DESIRED_STATE_CHANGED=YES_FINAL_SEAL_COMMIT_ONLY ALL_HAPUSYA_SITE_TAILS_CLOSED=YES TASK_COMPLETE=YES
1120:/var/lib/homelab-health/pve03-failed-unit-cleanup-closure.txt:1:STATUS=OK TS=2026-07-07T12:52:21Z TYPE=pve03-failed-unit-cleanup-closure BAD=0 WARN=0 MODE=READ_ONLY_CLOSURE LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CP_MATCH=YES RESTORE_CHECK=YES PVE03_FAILED_UNITS=0 REPORT=/var/lib/homelab-pve03-failed-unit-cleanup/closure-latest.md
1128:/var/lib/homelab-health/cluster-admin-backup-webpanel-final-seal.txt:1:STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[PRIVATE_IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md
1132:/var/lib/homelab-health/appbackupctl-routing.txt:1:STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z
1134:/var/lib/homelab-health/cluster-admin-stage3-api-apply.txt:1:STATUS=OK TS=2026-07-13T03:22:35Z TYPE=cluster-admin-stage3-api-apply BAD=0 COMMAND_PHYSICAL_LINES=1 RESULT=APPLIED PRODUCTION_CHANGED=LIVE_APP_AND_DESIRED_STATE PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=YES CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_PREAPPLY_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES PREAPPLY_BACKUP_REUSED=YES AST_IDENTICAL_NORMALIZATION=YES EXISTING_ROUTES_BASELINE=OK NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES OPENAPI_DOCS_EXPOSURE_CHANGED=NO CORE_DATABASE_STATE=1|1|9|7|1 INCIDENT_ENGINE=ACTIVE VM180_FAILED_UNITS=0 ROLLBACK_CHECKPOINT=VERIFIED OLD_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a NEW_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 READY_FOR_STAGE3_AUTHENTICATED_PRODUCTION_VERIFY=YES STAGE3_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-production-apply/20260713T032235Z
1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL
1146:/var/lib/homelab-health/special-vm-emergency-non-app-final-seal.txt:1:STATUS=OK TS=2026-07-14T01:15:48Z TYPE=special-vm-emergency-non-app-final-seal BAD=0 WARN=0 FAMILY_COVERAGE=11/11 MATRIX_STATUS=OK JOBS_TOTAL=40 JOBS_OK=36 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 COMPATIBILITY_OUTPUTS=31 REQUIRED_GREEN_JOBS=14 VM_FULL=OK VM_LOCAL_DUMPS=OK EDGE_VM_SOURCE=OK EDGE_VM_CLOUD=OK NEXTCLOUD_VM_SOURCE=homelab_vm_local_dumps_2cloud NEXTCLOUD_VM_CLOUD=OK EMERGENCY_PACK=OK SKLADCHIK_REPORTS_MONITOR=OK GOLDEN_STATE=OK ROUTER_RUNNING_CONFIG=OK MAIL_CLOUD_CRITICAL=OK MAIL_CLOUD_EDGE=OK MAIL_CLOUD_NEXTCLOUD=OK WRITE_MODE=legacy READ_MODE=dual MIGRATE_EXISTING=no SOURCE_BACKUP_RUN=NO CLOUD_OPERATIONS_RUN=NO RETENTION_RUN=NO CLOUD_PURGE_RUN=NO CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=b26f498b55de5d80bcbb4b9fb3a9bccded9f99b4 CURRENT_SPECIAL_VM_EMERGENCY_NON_APP_STAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/special-vm-emergency-non-app-final-seal/20260714T011514Z
1151:/var/lib/homelab-health/crypto-portfolio-ui-preapply-backup.txt:1:STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z
1157:/var/lib/homelab-health/legacy-monitoring-backups-migrated.txt:1:STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK
1159:/var/lib/homelab-health/cluster-admin-stage2-final-seal.txt:1:STATUS=OK TS=2026-07-13T01:45:29Z TYPE=cluster-admin-stage2-final-seal BAD=0 STAGE1_CLOSED=YES STAGE2_APPLY=OK PRODUCTION_CHANGED=BACKUP_HEALTH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_SCHEMA_CHANGED=NO_ADDITIONAL_CHANGE_DURING_SEAL INCIDENT_ENGINE_RESTARTED=NO MIGRATION_ID=002_canonical_foundation_sync FUNCTION=cluster_admin_sync_foundation COMMITTED_SYNC=YES MULTICYCLE_COLLECTOR_TEST=OK MULTICYCLE_NEW_RUNS=2 MULTICYCLE_GOOD_RUNS=2 ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK RETIRED_PLACEHOLDERS_ALLOWED=YES FOUNDATION_COUNTS=85|21|2|229|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0|0 ORPHANS=0 POSTAPPLY_DIRECT_SNAPSHOT=VERIFIED POSTAPPLY_RESTORE_TEST=OK POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T014757Z LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE2_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage2/stage2-final-seal-latest.txt REPORT_SHA256=32d4d37d82aa25194d789083d0e7ad2e9052ee38a72df5312b711226898cf1a6 BLOCKERS=/var/lib/homelab-cluster-admin-stage2/20260713T014529Z/blockers.txt
1169:/var/lib/homelab-health/cluster-admin-stage3-final-seal.txt:1:STATUS=OK TS=2026-07-13T03:46:30Z TYPE=cluster-admin-stage3-final-seal BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=HEALTH_REFRESH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO STAGE2_CLOSED=YES STAGE3_APPLY=OK AUTHENTICATED_PRODUCTION_VERIFY=OK AUTHENTICATED_PRODUCTION_TESTS=25 APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 NEW_GET_ROUTES=7 DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY OPENAPI_DOCS_EXPOSURE=PRESERVED_DISABLED POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z POSTAPPLY_APPBACKUP_RESTORE_CHECK=OK COLLECTOR_STABILITY=OK LATEST_COLLECTOR_RUN_ID=3114 CANONICAL_CONSISTENCY=0|0|0|0|0 OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE3_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage3/stage3-final-seal-latest.txt REPORT_SHA256=6f67545cd7f0a75b87146c1df74c61c3b88e142f9e53a96b083ec27209e6d265
1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL
1172:/var/lib/homelab-health/backup-matrix-desired-state-persistence.txt:1:STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z
1175:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES
1184:/var/lib/homelab-health/cluster-admin-stage4d-adapter-candidates-in-isolation.txt:1:STATUS=OK TS=2026-07-14T06:14:58Z TYPE=cluster-admin-stage4d-adapter-candidates-in-isolation-final-seal BAD=0 STAGE4D_CLOSED=YES NEXT_STAGE=4E_RESTRICTED_PROBE_AGENT_DESIGN PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO APPLICATION_CHANGED=NO ENVELOPE_TOTAL=48 PROMETHEUS_ENVELOPES=5 BACKUP_MATRIX_ENVELOPES=40 HEALTH_ENVELOPES=3 LOCAL_REMOTE_OUTPUT_MATCH=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=e77fb37a0fa0033fe46558045e31dd92b6d1238a DESIRED_STATE_CLEAN=YES PVE01_FAILED_UNITS=0 REPORT=/var/lib/homelab-cluster-admin-stage4/20260714T061458Z/stage4d-adapter-candidates-in-isolation-seal.txt REPORT_SHA256=da6244c0a8a458b508fd58a0fec9cb1e6fa8da98a36f44d1368cbe947f4b5835
1185:/var/lib/homelab-health/backup-matrix-cluster-desired-state-sync.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
1187:/var/lib/homelab-health/cluster-admin-stage3-api-runtime-dry-run.txt:1:STATUS=OK TS=2026-07-13T02:53:34Z TYPE=cluster-admin-stage3-api-runtime-dry-run BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=TEMPORARY_ISOLATED_RUNTIME_AND_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO PRODUCTION_SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ISOLATED_BIND=127.0.0.1:18081 TEMPORARY_DATABASE_RESTORED=YES TEMPORARY_DATABASE_DROPPED=YES TEMPORARY_DATABASE_UNCHANGED=YES RUNTIME_TESTS=41 RUNTIME_FAILED_TESTS=0 EXISTING_ROUTES_BACKWARD_COMPATIBLE=YES NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES AUTHENTICATED_NEW_ROUTES=OK PAGINATION=OK FILTER_VALIDATION=OK ENTITY_DETAIL=OK ENTITY_DEPENDENCIES=OK SOURCES=OK POLICIES=OK COLLECTOR_RUNS=OK DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY FORBIDDEN_RESPONSE_FIELDS=ABSENT OPENAPI_DOCS_EXPOSURE_CHANGED=NO SECURITY_HEADERS=OK ISOLATED_RUNTIME_STOPPED=YES ISOLATED_PORT_CLOSED=YES LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK READY_FOR_STAGE3_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z RESULT=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z/runtime-result.json
1188:/var/lib/homelab-health/crypto-portfolio-remove-grt-failed-units-diagnostic.txt:1:STATUS=OK TS=2026-07-13T05:41:00Z TYPE=crypto-portfolio-remove-grt-failed-units-diagnostic BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=NO_DIAGNOSTIC_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO FAILED_STATE_RESET=NO CLOUD_WRITE_RUN=NO PVE01_FAILED_UNIT_COUNT=2 PVE01_FAILED_UNIT_NAMES=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service CRYPTO_PORTFOLIO_FAILED_UNIT_COUNT=0 OTHER_FAILED_UNIT_COUNT=2 CURRENT_CRYPTO_PRECHANGE_HEALTH=YES UNIT_DETAILS_CAPTURED=YES READY_FOR_TARGETED_FAILED_UNIT_DECISION=YES DIAGNOSTIC_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-failed-units-diagnostic/20260713T054100Z
1189:/var/lib/homelab-health/crypto-portfolio-remove-grt-production-apply.txt:1:STATUS=OK TS=2026-07-13T05:47:54Z TYPE=crypto-portfolio-remove-grt-production-apply BAD=0 COMMAND_PHYSICAL_LINES=1 APPLY_REVISION=V3 FIX_UID_READONLY=YES FIX_READINESS_ENDPOINT=YES FIX_ROLLBACK_DATABASE_METADATA=YES PRODUCTION_CHANGED=YES_REMOVE_GRT DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND PRODUCTION_CONTAINER_RECREATED=YES CLOUD_WRITE_RUN=NO ROLLBACK_ON_FAILURE=ARMED_NOT_NEEDED TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 PRECHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CHART_HEALTH_SCRIPT_SHA256=c16473f16390eeeff4d4eb701db604093564dd78d0be2d30b4cc8ffeeb22364a ACTIVE_COINS=HBAR,POL GRT_SOURCE_REFERENCES=0 GRT_DATABASE_ROWS=0|0|0 GRT_MARKET_ENTRY=ABSENT GRT_CACHE_ENTRIES=ABSENT DATABASE_MODE=0644 DATABASE_UID=10001 DATABASE_GID=10001 DATABASE_INTEGRITY=ok WORKER_STATUS=OK WORKER_COIN_COUNT=2 PUBLIC_CODES=200|303|401|200|200|401 CHART_HEALTH=OK CHART_SYMBOLS=HBAR,POL CHART_HEALTH_TIMER=ACTIVE VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_UNITS=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES REMOTE_STAGE_CLEANED=YES READY_FOR_GRT_POSTAPPLY_VERIFY_BACKUP_SYNC=YES APPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-production-apply-v3/20260713T054754Z
1193:/var/lib/homelab-health/crypto-portfolio-runtime-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES
1194:/var/lib/homelab-health/backup-matrix-cluster-preseal-convergence.txt:1:STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z
1202:/var/lib/homelab-health/node-loss-final-closure.txt:1:STATUS=OK TS=2026-07-08T16:35:56Z TYPE=node-loss-final-closure BAD=0 MODE=ALL_NODE_LOSS_DRILLS_SEALED PVE03_EDGE_RESTORE=OK PVE03_EDGE_FRESHNESS=OK PVE02_FORUM_RUNTIME=OK PVE01_NEXTCLOUD_DNS=OK PVE01_CORE_MONITORING_APPLEVEL=OK FULL_VM_GAP_VM170_VM171=ACKNOWLEDGED_APP_LEVEL_ONLY TEST_VMS_CLEANED=YES DNS_PAIR_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-node-loss/final-closure-latest.md
1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed
1332:/var/lib/homelab-health/codevipe-cloud-backup-reset.txt:1:STATUS=OK TS=2026-07-08T05:25:22Z TYPE=remote-failed-unit-codevipe-reset BAD=0 WARN=0 MODE=RESET_STALE_FAILED_STATE UNIT=codevipe-cloud-backup.service REASON=last_journal_success_and_no_timer ACTION=reset_failed_only
1426:● homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image
1499:2026-07-21T11:35:23Z	BACKUP-DUAL-CLOUD-PREFLIGHT-059	c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479	1	FAIL	backup-audit
1500:2026-07-21T12:28:26Z	BACKUP-VM-DUMP-PIPELINE-073	cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b	1	FAIL	backup-audit
1501:2026-07-21T12:42:53Z	BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075	a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936	143	FAIL	backup-audit
1502:2026-07-21T12:56:52Z	BACKUP-DRAIN-LOCAL-ARCHIVES-079	531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32	75	FAIL	backup-remediation
1503:2026-07-21T13:04:48Z	BACKUP-FINAL-PROOF-082	9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601	127	FAIL	backup-audit
1505:2026-07-21T13:14:27Z	CLUSTER-BACKUP-POSTCLEAN-PROOF-086	601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da	127	FAIL	backup-cleanup
1506:2026-07-21T13:15:21Z	BACKUP-POLICY-CLOSEOUT-087	d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52	1	FAIL	backup-cleanup
1507:2026-07-21T13:17:20Z	BACKUP-CLEANUP-FINAL-CLOSEOUT-088	5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6	1	FAIL	backup-cleanup
=== ACTIONABLE_P1 ===
=== ACTIONABLE_P1 ===
=== ACTIONABLE_P1 ===
26:1:# HOMELAB ASSISTANT ERROR REGISTER
28:77:21. Ошибка: nested PHP php -r дал Parse error на forum-prod.
29:83:Команда 108 дала PHP Parse error на smtpHost и пустой proof-файл.
30:88:Команда 111 не изменила smtpSsl: OLD_SMTPSSL=false, SQL ERROR 1064, NEW_SMTPSSL=false.
40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.
41:154:37. Coverage gap: pve03_staging missing from disk-space health coverage.
42:159:Факт: terminal output around command 222 showed paste artifact near "FAIL; fi".
45:171:Факт: first reconciliation wrote STATUS=ERROR because router-running-config-mail-cloud.txt is on pve01, not edge-vm.
51:194:- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200.
52:195:- If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed.
55:199:- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('.
60:227:- Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli].
79:368:- Evidence: proof 542 showed all five zones missing and DNS record create probes failed.
83:388:- Impact: web routing is OK, but mail-related DNS may still contain stale provider data.
84:392:- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt.
85:397:- Context: SPF cleanup command 565 attempted to replace stale SPF records.
93:416:- Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line.
95:429:- Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth.
97:435:- Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory contains old codevipe-public/codevipe.sql files, not current five-forum archive.
120:621:- Certbot on edge rejected --dns-cloudflare-* arguments: dns-cloudflare plugin missing/not loaded.
129:657:- Факт: bash начал интерпретировать Python-код, появились syntax error и вторичный prompt >.
131:675:- When asked to fix one missing Uptime Kuma service, add only one monitor, then generate proposals separately.
134:681:- Dockge inactive items after migration can be stale compose folders, not stopped containers.
135:684:- Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions.
145:737:- The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults.
149:761:- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults.
153:774:- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects.
155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.
171:839:- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`.
180:865:- Symptom: error-register candidate construction stopped with RC1 before applying the candidate.
183:876:- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis.
201:948:BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative.
203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.
220:- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR.
235:- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring.
236:- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification.
245:- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first.
250:- Stale moved edge stack folders archived under /opt/dockge-stale-archive/20260702T230442Z on edge-vm; no runtime containers were deleted.
252:- Final target: remote not_visible count must be 0 and edge stale remaining matches must be 0.
258:- Stale moved stacks were archived, not deleted.
316:- Фактические поля collector_runs: finished_at и error_text; completed_at и error отсутствуют.
320:- Production осталась 0|0|OK, timer активен, failed units отсутствуют.
372:    "missing": [
459:STATUS=FAIL
475:JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000
486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
495:JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000
508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
510:JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000
513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
676:2026-07-08T09:26:36.7403618890 390 /var/lib/homelab-health/dockge-stale-stack-classification-after-agents.txt
681:2026-07-08T09:26:36.7406026680 390 /var/lib/homelab-health/homepage-missing-card-proposals.txt
684:2026-07-08T09:26:36.7408259230 390 /var/lib/homelab-health/homepage-missing-card-proposals-final.txt
689:2026-07-08T09:26:36.7410552890 389 /var/lib/homelab-health/dockge-stale-archive-last.txt
1114:/var/lib/homelab-health/service-registry.txt:1:STATUS=OK TS=2026-07-21T13:15:17Z TYPE=service-registry SERVICES=12 MISSING=0 REGISTRY=/var/lib/homelab-registry/service-registry-latest.tsv
1122:/var/lib/homelab-health/nextcloud-restore-proof-offhost.txt:1:STATUS=OK TS=20260624T190330Z TYPE=nextcloud-restore-proof-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge/nextcloud-aio/nextcloud-aio-restore-proof/20260624T120344Z ITEMS=1 OK=1 ERROR=0 BYTES=2135684 FILES=7 SECRET_PRINTED=NO
1127:/var/lib/homelab-health/crypto-portfolio-remove-grt-recovery-audit.txt:1:STATUS=ERROR TS=2026-07-13T05:39:07Z TYPE=crypto-portfolio-remove-grt-recovery-audit BAD=1 COMMAND_PHYSICAL_LINES=1 ROOT_CAUSE=READINESS_ENDPOINT_MISMATCH RECOVERY_VERIFIED=NO TASK_COMPLETE=NO READY_FOR_GRT_APPLY_SCRIPT_FIX=NO AUDIT_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-recovery-audit-v2/20260713T053907Z
1133:/var/lib/homelab-health/final-readiness.txt:1:STATUS=WARN TS=2026-07-21T05:31:30Z TYPE=final-readiness BAD=5 PASS=48 FAIL=5 REPORT=/var/lib/homelab-final-readiness/latest.md
1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL
1152:/var/lib/homelab-health/router-hard-power.txt:1:STATUS=ERROR TS=2026-07-16T06:20:34Z MODE=cycle REMOTE=[PRIVATE_IP] RC=74
1162:/var/lib/homelab-health/private-vpn-hosts.txt.bak-ok-wrapper-20260624T132705Z:16:STATUS=ERROR
1164:/var/lib/homelab-health/paperless-restore.txt:1:STATUS=STALE
1165:/var/lib/homelab-health/paperless-restore.txt:20:FRESHNESS=STALE
1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL
1174:/var/lib/homelab-health/ungated-health-backlog.txt:1:STATUS=OK TS=2026-07-07T07:45:01Z TYPE=ungated-health-backlog BAD=0 WARN=1 MODE=POST_PVE03_ROOT_CLEANUP_TRIAGE TOTAL=8 SUPERSEDED=4 STALE=2 POLICY_GAP_NEXT_STAGE=1 ACTIONABLE_CAPACITY=1 PVE03_ROOT_USAGE_PERCENT=31 PVE03_STAGING_USAGE_PERCENT=83 REPORT=/var/lib/homelab-ungated-health-backlog/latest.md
1190:/var/lib/homelab-health/mail-cloud-restore-drill.txt:1:STATUS=STALE
1191:/var/lib/homelab-health/mail-cloud-restore-drill.txt:20:FRESHNESS=STALE
1192:/var/lib/homelab-health/edge-restore-proofs-offhost.txt:1:STATUS=OK TS=20260624T190040Z TYPE=edge-restore-proofs-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge ITEMS=14 OK=14 ERROR=0 BYTES=226024287 SECRET_PRINTED=NO
1195:/var/lib/homelab-health/uptime-kuma-pending-health-monitors-fix.txt:8:NOTE=Paperless Restore may become red if its health file contains STATUS=ERROR; that is a real restore-health issue, not a Kuma pending issue.
1199:/var/lib/homelab-health/cluster-admin-stage0-convergence.txt:1:STATUS=WARN TS=2026-07-12T23:12:59Z TYPE=cluster-admin-stage0-targeted-convergence BAD=2 PROFILE_OK=NO CLUSTER_PASSPORT=FAIL FINAL_READINESS=FAIL OVERALL=FAIL DUTY_ADMIN=FAIL FAILED_UNITS=0 LIVE_MATCH_DESIRED_STATE=YES READY_FOR_FINAL_SEAL=NO REPORT_DIR=/var/lib/homelab-cluster-admin-stage0/readiness-profile-20260712T231259Z PRODUCTION_CHANGED=HEALTH_REGENERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO
1207:/var/lib/homelab-health/filebrowser-restore.txt:1:STATUS=STALE
1208:/var/lib/homelab-health/filebrowser-restore.txt:20:FRESHNESS=STALE
1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed
1494:2026-07-21T08:03:07Z	GITEA-PUSH-AUTH-001	fcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94	1	FAIL	gitea-runtime
1495:2026-07-21T08:13:32Z	GITEA-TOKEN-CLI-HELP-003	f1d0f967eed6490ad822944a42108c82dc4b59c9fd2557d8c64624dcb6853cd4	1	FAIL	gitea-runtime
1496:2026-07-21T08:14:52Z	GITEA-CLI-CAPABILITIES-004	2cd6c4d28c8f646d8063105fca80b2f80d38c6a5db3a3eb76869de326b12d2d6	1	FAIL	gitea-runtime
1497:2026-07-21T09:49:51Z	RUNTIME-RECEIVER-STATE-027	0831002c78c26f35826cd7199995e335ceadcd39f08b9bac23882a0fbb59c8cc	127	FAIL	gitea-runtime
1498:2026-07-21T10:26:18Z	RUNTIME-RECEIVER-CLEANUP-036	65fbcfbce38a55723a92a8790429d6567389321f030f02cb4bf301cbe67e2531	1	FAIL	gitea-runtime
1499:2026-07-21T11:35:23Z	BACKUP-DUAL-CLOUD-PREFLIGHT-059	c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479	1	FAIL	backup-audit
1500:2026-07-21T12:28:26Z	BACKUP-VM-DUMP-PIPELINE-073	cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b	1	FAIL	backup-audit
1501:2026-07-21T12:42:53Z	BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075	a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936	143	FAIL	backup-audit
1502:2026-07-21T12:56:52Z	BACKUP-DRAIN-LOCAL-ARCHIVES-079	531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32	75	FAIL	backup-remediation
1503:2026-07-21T13:04:48Z	BACKUP-FINAL-PROOF-082	9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601	127	FAIL	backup-audit
1504:2026-07-21T13:07:09Z	CLUSTER-CLEANUP-INVENTORY-083	09d3f14ca0205936a910dc7af7991474030273a47d76fa033ea6a3185c083922	1	FAIL	cluster-cleanup
1505:2026-07-21T13:14:27Z	CLUSTER-BACKUP-POSTCLEAN-PROOF-086	601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da	127	FAIL	backup-cleanup
1506:2026-07-21T13:15:21Z	BACKUP-POLICY-CLOSEOUT-087	d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52	1	FAIL	backup-cleanup
1507:2026-07-21T13:17:20Z	BACKUP-CLEANUP-FINAL-CLOSEOUT-088	5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6	1	FAIL	backup-cleanup
=== ACTIONABLE_STORAGE ===
=== ACTIONABLE_STORAGE ===
=== ACTIONABLE_STORAGE ===
40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.
454:local             dir     active       102626232        41916788        55450180   40.84%
=== OPEN_ERROR_REGISTER ===
=== FIRST_PRIORITY_CANDIDATES ===
38-BOOTSTRAP_SECTION_LINES=0
39-=== ACTIONABLE_P0 ===
40:86:398:- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained.
41:155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.
42:203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.
43:486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
44:508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
45:513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
46:546:● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state
47:548:● homelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03
48:1102:/var/lib/homelab-health/backup-matrix-cluster-final-seal.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
49:1104:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-production-closure.txt:1:STATUS=OK TS=2026-07-13T08:12:14Z TYPE=crypto-portfolio-ui-center-alert-currency-production-closure BAD=0 PRODUCTION_CHANGED=YES_UI_ONLY INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 POSTCHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION_ROLLBACK_COUNT=3 RETENTION_OPTIONAL_OTHER_CANDIDATES=1 RETENTION_REMOVED_ROLLBACK_COUNT=0 DANGLING_IMAGES_REMAINING=2 UNREFERENCED_DANGLING_IMAGES=0 DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE_CLEAN=YES PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES READY_FOR_UI_FINAL_SEAL=YES
50:1106:/var/lib/homelab-health/cluster-admin-stage3-postapply-backup.txt:1:STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z
51:1109:/var/lib/homelab-health/pve01-core-monitoring-app-restorecheck.txt:1:STATUS=OK TS=2026-07-08T15:57:27Z TYPE=pve01-core-monitoring-app-restorecheck BAD=0 MODE=APP_LEVEL_RESTORE_CHECKS_TWO_CLOUDS FULL_VM_RESTORE_AVAILABLE=NO TOTAL_APPS=30 OK_APPS=30 FAIL_LIST=none REPORT=/var/lib/homelab-node-loss/pve01-core-monitoring-app-level/app-restorecheck-latest.md
52:1118:/var/lib/homelab-health/hapusya-site-final-seal.txt:1:STATUS=OK TS=2026-07-13T12:33:08Z TYPE=hapusya-site-final-seal BAD=0 APP=hapusya-site URL=https://hapusya.ru/ PRESEAL_DESIRED_STATE_HEAD=69966eb83fc065141df3c46df0d6c0f282275046 ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb APPLY_MANIFEST_SHA256=f44992e3e0cc443b372779cb35d9e899d904a9960a9de03c14559886fdb1fc75 PRODUCTION_BROWSER_MANIFEST_SHA256=9f3cdf3063f8724d1d1c1ad619f47d6f98d434c3422a74903585146858a8dfac BROWSER_SMOKE=OK BROWSER_RUNS=5 BROWSER_ERRORS=0 DESKTOP_VIDEO=OK MOBILE_VIDEO=OK STATIC_FALLBACK=OK BACKUP_JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 TWO_CLOUDS=OK RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK DESIRED_STATE_SYNC=OK GENERATOR_SHA256=507bbbc2691f1f08cb1dd3660ed76cf13a8c36c23833d5fdec40cf6e07486cb8 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 POLICY_SHA256=4bf996ff451ae3ec56eb2fcd42842606cd97295b4d5141aedb135c956395e4b5 EVIDENCE_ROOT=/var/lib/homelab-change-sets/hapusya-site-final-seal/20260713T123308Z EVIDENCE_MANIFEST_SHA256=7eb993a17ac18a1f7c9ae0b52d745d3778833d15626ed34c82496f74916eb26a PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY DESIRED_STATE_CHANGED=YES_FINAL_SEAL_COMMIT_ONLY ALL_HAPUSYA_SITE_TAILS_CLOSED=YES TASK_COMPLETE=YES
53:1120:/var/lib/homelab-health/pve03-failed-unit-cleanup-closure.txt:1:STATUS=OK TS=2026-07-07T12:52:21Z TYPE=pve03-failed-unit-cleanup-closure BAD=0 WARN=0 MODE=READ_ONLY_CLOSURE LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CP_MATCH=YES RESTORE_CHECK=YES PVE03_FAILED_UNITS=0 REPORT=/var/lib/homelab-pve03-failed-unit-cleanup/closure-latest.md
54:1128:/var/lib/homelab-health/cluster-admin-backup-webpanel-final-seal.txt:1:STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[PRIVATE_IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md
55:1132:/var/lib/homelab-health/appbackupctl-routing.txt:1:STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z
56:1134:/var/lib/homelab-health/cluster-admin-stage3-api-apply.txt:1:STATUS=OK TS=2026-07-13T03:22:35Z TYPE=cluster-admin-stage3-api-apply BAD=0 COMMAND_PHYSICAL_LINES=1 RESULT=APPLIED PRODUCTION_CHANGED=LIVE_APP_AND_DESIRED_STATE PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=YES CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_PREAPPLY_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES PREAPPLY_BACKUP_REUSED=YES AST_IDENTICAL_NORMALIZATION=YES EXISTING_ROUTES_BASELINE=OK NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES OPENAPI_DOCS_EXPOSURE_CHANGED=NO CORE_DATABASE_STATE=1|1|9|7|1 INCIDENT_ENGINE=ACTIVE VM180_FAILED_UNITS=0 ROLLBACK_CHECKPOINT=VERIFIED OLD_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a NEW_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 READY_FOR_STAGE3_AUTHENTICATED_PRODUCTION_VERIFY=YES STAGE3_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-production-apply/20260713T032235Z
57:1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL
58:1146:/var/lib/homelab-health/special-vm-emergency-non-app-final-seal.txt:1:STATUS=OK TS=2026-07-14T01:15:48Z TYPE=special-vm-emergency-non-app-final-seal BAD=0 WARN=0 FAMILY_COVERAGE=11/11 MATRIX_STATUS=OK JOBS_TOTAL=40 JOBS_OK=36 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 COMPATIBILITY_OUTPUTS=31 REQUIRED_GREEN_JOBS=14 VM_FULL=OK VM_LOCAL_DUMPS=OK EDGE_VM_SOURCE=OK EDGE_VM_CLOUD=OK NEXTCLOUD_VM_SOURCE=homelab_vm_local_dumps_2cloud NEXTCLOUD_VM_CLOUD=OK EMERGENCY_PACK=OK SKLADCHIK_REPORTS_MONITOR=OK GOLDEN_STATE=OK ROUTER_RUNNING_CONFIG=OK MAIL_CLOUD_CRITICAL=OK MAIL_CLOUD_EDGE=OK MAIL_CLOUD_NEXTCLOUD=OK WRITE_MODE=legacy READ_MODE=dual MIGRATE_EXISTING=no SOURCE_BACKUP_RUN=NO CLOUD_OPERATIONS_RUN=NO RETENTION_RUN=NO CLOUD_PURGE_RUN=NO CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=b26f498b55de5d80bcbb4b9fb3a9bccded9f99b4 CURRENT_SPECIAL_VM_EMERGENCY_NON_APP_STAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/special-vm-emergency-non-app-final-seal/20260714T011514Z
59:1151:/var/lib/homelab-health/crypto-portfolio-ui-preapply-backup.txt:1:STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z
60:1157:/var/lib/homelab-health/legacy-monitoring-backups-migrated.txt:1:STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK
61:1159:/var/lib/homelab-health/cluster-admin-stage2-final-seal.txt:1:STATUS=OK TS=2026-07-13T01:45:29Z TYPE=cluster-admin-stage2-final-seal BAD=0 STAGE1_CLOSED=YES STAGE2_APPLY=OK PRODUCTION_CHANGED=BACKUP_HEALTH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_SCHEMA_CHANGED=NO_ADDITIONAL_CHANGE_DURING_SEAL INCIDENT_ENGINE_RESTARTED=NO MIGRATION_ID=002_canonical_foundation_sync FUNCTION=cluster_admin_sync_foundation COMMITTED_SYNC=YES MULTICYCLE_COLLECTOR_TEST=OK MULTICYCLE_NEW_RUNS=2 MULTICYCLE_GOOD_RUNS=2 ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK RETIRED_PLACEHOLDERS_ALLOWED=YES FOUNDATION_COUNTS=85|21|2|229|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0|0 ORPHANS=0 POSTAPPLY_DIRECT_SNAPSHOT=VERIFIED POSTAPPLY_RESTORE_TEST=OK POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T014757Z LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE2_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage2/stage2-final-seal-latest.txt REPORT_SHA256=32d4d37d82aa25194d789083d0e7ad2e9052ee38a72df5312b711226898cf1a6 BLOCKERS=/var/lib/homelab-cluster-admin-stage2/20260713T014529Z/blockers.txt
62:1169:/var/lib/homelab-health/cluster-admin-stage3-final-seal.txt:1:STATUS=OK TS=2026-07-13T03:46:30Z TYPE=cluster-admin-stage3-final-seal BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=HEALTH_REFRESH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO STAGE2_CLOSED=YES STAGE3_APPLY=OK AUTHENTICATED_PRODUCTION_VERIFY=OK AUTHENTICATED_PRODUCTION_TESTS=25 APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 NEW_GET_ROUTES=7 DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY OPENAPI_DOCS_EXPOSURE=PRESERVED_DISABLED POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z POSTAPPLY_APPBACKUP_RESTORE_CHECK=OK COLLECTOR_STABILITY=OK LATEST_COLLECTOR_RUN_ID=3114 CANONICAL_CONSISTENCY=0|0|0|0|0 OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE3_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage3/stage3-final-seal-latest.txt REPORT_SHA256=6f67545cd7f0a75b87146c1df74c61c3b88e142f9e53a96b083ec27209e6d265
63:1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL
64:1172:/var/lib/homelab-health/backup-matrix-desired-state-persistence.txt:1:STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z
65:1175:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES
66:1184:/var/lib/homelab-health/cluster-admin-stage4d-adapter-candidates-in-isolation.txt:1:STATUS=OK TS=2026-07-14T06:14:58Z TYPE=cluster-admin-stage4d-adapter-candidates-in-isolation-final-seal BAD=0 STAGE4D_CLOSED=YES NEXT_STAGE=4E_RESTRICTED_PROBE_AGENT_DESIGN PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO APPLICATION_CHANGED=NO ENVELOPE_TOTAL=48 PROMETHEUS_ENVELOPES=5 BACKUP_MATRIX_ENVELOPES=40 HEALTH_ENVELOPES=3 LOCAL_REMOTE_OUTPUT_MATCH=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=e77fb37a0fa0033fe46558045e31dd92b6d1238a DESIRED_STATE_CLEAN=YES PVE01_FAILED_UNITS=0 REPORT=/var/lib/homelab-cluster-admin-stage4/20260714T061458Z/stage4d-adapter-candidates-in-isolation-seal.txt REPORT_SHA256=da6244c0a8a458b508fd58a0fec9cb1e6fa8da98a36f44d1368cbe947f4b5835
67:1185:/var/lib/homelab-health/backup-matrix-cluster-desired-state-sync.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
68:1187:/var/lib/homelab-health/cluster-admin-stage3-api-runtime-dry-run.txt:1:STATUS=OK TS=2026-07-13T02:53:34Z TYPE=cluster-admin-stage3-api-runtime-dry-run BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=TEMPORARY_ISOLATED_RUNTIME_AND_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO PRODUCTION_SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ISOLATED_BIND=127.0.0.1:18081 TEMPORARY_DATABASE_RESTORED=YES TEMPORARY_DATABASE_DROPPED=YES TEMPORARY_DATABASE_UNCHANGED=YES RUNTIME_TESTS=41 RUNTIME_FAILED_TESTS=0 EXISTING_ROUTES_BACKWARD_COMPATIBLE=YES NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES AUTHENTICATED_NEW_ROUTES=OK PAGINATION=OK FILTER_VALIDATION=OK ENTITY_DETAIL=OK ENTITY_DEPENDENCIES=OK SOURCES=OK POLICIES=OK COLLECTOR_RUNS=OK DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY FORBIDDEN_RESPONSE_FIELDS=ABSENT OPENAPI_DOCS_EXPOSURE_CHANGED=NO SECURITY_HEADERS=OK ISOLATED_RUNTIME_STOPPED=YES ISOLATED_PORT_CLOSED=YES LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK READY_FOR_STAGE3_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z RESULT=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z/runtime-result.json
69:1188:/var/lib/homelab-health/crypto-portfolio-remove-grt-failed-units-diagnostic.txt:1:STATUS=OK TS=2026-07-13T05:41:00Z TYPE=crypto-portfolio-remove-grt-failed-units-diagnostic BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=NO_DIAGNOSTIC_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO FAILED_STATE_RESET=NO CLOUD_WRITE_RUN=NO PVE01_FAILED_UNIT_COUNT=2 PVE01_FAILED_UNIT_NAMES=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service CRYPTO_PORTFOLIO_FAILED_UNIT_COUNT=0 OTHER_FAILED_UNIT_COUNT=2 CURRENT_CRYPTO_PRECHANGE_HEALTH=YES UNIT_DETAILS_CAPTURED=YES READY_FOR_TARGETED_FAILED_UNIT_DECISION=YES DIAGNOSTIC_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-failed-units-diagnostic/20260713T054100Z
70:1189:/var/lib/homelab-health/crypto-portfolio-remove-grt-production-apply.txt:1:STATUS=OK TS=2026-07-13T05:47:54Z TYPE=crypto-portfolio-remove-grt-production-apply BAD=0 COMMAND_PHYSICAL_LINES=1 APPLY_REVISION=V3 FIX_UID_READONLY=YES FIX_READINESS_ENDPOINT=YES FIX_ROLLBACK_DATABASE_METADATA=YES PRODUCTION_CHANGED=YES_REMOVE_GRT DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND PRODUCTION_CONTAINER_RECREATED=YES CLOUD_WRITE_RUN=NO ROLLBACK_ON_FAILURE=ARMED_NOT_NEEDED TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 PRECHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CHART_HEALTH_SCRIPT_SHA256=c16473f16390eeeff4d4eb701db604093564dd78d0be2d30b4cc8ffeeb22364a ACTIVE_COINS=HBAR,POL GRT_SOURCE_REFERENCES=0 GRT_DATABASE_ROWS=0|0|0 GRT_MARKET_ENTRY=ABSENT GRT_CACHE_ENTRIES=ABSENT DATABASE_MODE=0644 DATABASE_UID=10001 DATABASE_GID=10001 DATABASE_INTEGRITY=ok WORKER_STATUS=OK WORKER_COIN_COUNT=2 PUBLIC_CODES=200|303|401|200|200|401 CHART_HEALTH=OK CHART_SYMBOLS=HBAR,POL CHART_HEALTH_TIMER=ACTIVE VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_UNITS=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES REMOTE_STAGE_CLEANED=YES READY_FOR_GRT_POSTAPPLY_VERIFY_BACKUP_SYNC=YES APPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-production-apply-v3/20260713T054754Z
71:1193:/var/lib/homelab-health/crypto-portfolio-runtime-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES
72:1194:/var/lib/homelab-health/backup-matrix-cluster-preseal-convergence.txt:1:STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z
73:1202:/var/lib/homelab-health/node-loss-final-closure.txt:1:STATUS=OK TS=2026-07-08T16:35:56Z TYPE=node-loss-final-closure BAD=0 MODE=ALL_NODE_LOSS_DRILLS_SEALED PVE03_EDGE_RESTORE=OK PVE03_EDGE_FRESHNESS=OK PVE02_FORUM_RUNTIME=OK PVE01_NEXTCLOUD_DNS=OK PVE01_CORE_MONITORING_APPLEVEL=OK FULL_VM_GAP_VM170_VM171=ACKNOWLEDGED_APP_LEVEL_ONLY TEST_VMS_CLEANED=YES DNS_PAIR_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-node-loss/final-closure-latest.md
74:1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed
75:1332:/var/lib/homelab-health/codevipe-cloud-backup-reset.txt:1:STATUS=OK TS=2026-07-08T05:25:22Z TYPE=remote-failed-unit-codevipe-reset BAD=0 WARN=0 MODE=RESET_STALE_FAILED_STATE UNIT=codevipe-cloud-backup.service REASON=last_journal_success_and_no_timer ACTION=reset_failed_only
76-1426:● homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image
77:1499:2026-07-21T11:35:23Z	BACKUP-DUAL-CLOUD-PREFLIGHT-059	c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479	1	FAIL	backup-audit
78:1500:2026-07-21T12:28:26Z	BACKUP-VM-DUMP-PIPELINE-073	cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b	1	FAIL	backup-audit
79:1501:2026-07-21T12:42:53Z	BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075	a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936	143	FAIL	backup-audit
80:1502:2026-07-21T12:56:52Z	BACKUP-DRAIN-LOCAL-ARCHIVES-079	531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32	75	FAIL	backup-remediation
81:1503:2026-07-21T13:04:48Z	BACKUP-FINAL-PROOF-082	9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601	127	FAIL	backup-audit
82:1505:2026-07-21T13:14:27Z	CLUSTER-BACKUP-POSTCLEAN-PROOF-086	601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da	127	FAIL	backup-cleanup
83:1506:2026-07-21T13:15:21Z	BACKUP-POLICY-CLOSEOUT-087	d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52	1	FAIL	backup-cleanup
84:1507:2026-07-21T13:17:20Z	BACKUP-CLEANUP-FINAL-CLOSEOUT-088	5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6	1	FAIL	backup-cleanup
85-=== ACTIONABLE_P1 ===
86-26:1:# HOMELAB ASSISTANT ERROR REGISTER
87-28:77:21. Ошибка: nested PHP php -r дал Parse error на forum-prod.
88-29:83:Команда 108 дала PHP Parse error на smtpHost и пустой proof-файл.
--
111-149:761:- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults.
112-153:774:- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects.
113:155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.
114-171:839:- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`.
115-180:865:- Symptom: error-register candidate construction stopped with RC1 before applying the candidate.
116-183:876:- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis.
117-201:948:BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative.
118:203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.
119-220:- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR.
120-235:- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring.
121-236:- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification.
122-245:- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first.
--
129-459:STATUS=FAIL
130-475:JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000
131:486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
132-495:JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000
133:508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
134-510:JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000
135:513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
136-676:2026-07-08T09:26:36.7403618890 390 /var/lib/homelab-health/dockge-stale-stack-classification-after-agents.txt
137-681:2026-07-08T09:26:36.7406026680 390 /var/lib/homelab-health/homepage-missing-card-proposals.txt
138-684:2026-07-08T09:26:36.7408259230 390 /var/lib/homelab-health/homepage-missing-card-proposals-final.txt
139-689:2026-07-08T09:26:36.7410552890 389 /var/lib/homelab-health/dockge-stale-archive-last.txt
--
142-1127:/var/lib/homelab-health/crypto-portfolio-remove-grt-recovery-audit.txt:1:STATUS=ERROR TS=2026-07-13T05:39:07Z TYPE=crypto-portfolio-remove-grt-recovery-audit BAD=1 COMMAND_PHYSICAL_LINES=1 ROOT_CAUSE=READINESS_ENDPOINT_MISMATCH RECOVERY_VERIFIED=NO TASK_COMPLETE=NO READY_FOR_GRT_APPLY_SCRIPT_FIX=NO AUDIT_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-recovery-audit-v2/20260713T053907Z
143-1133:/var/lib/homelab-health/final-readiness.txt:1:STATUS=WARN TS=2026-07-21T05:31:30Z TYPE=final-readiness BAD=5 PASS=48 FAIL=5 REPORT=/var/lib/homelab-final-readiness/latest.md
144:1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL
145-1152:/var/lib/homelab-health/router-hard-power.txt:1:STATUS=ERROR TS=2026-07-16T06:20:34Z MODE=cycle REMOTE=[PRIVATE_IP] RC=74
146-1162:/var/lib/homelab-health/private-vpn-hosts.txt.bak-ok-wrapper-20260624T132705Z:16:STATUS=ERROR
147-1164:/var/lib/homelab-health/paperless-restore.txt:1:STATUS=STALE
148-1165:/var/lib/homelab-health/paperless-restore.txt:20:FRESHNESS=STALE
149:1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL
150-1174:/var/lib/homelab-health/ungated-health-backlog.txt:1:STATUS=OK TS=2026-07-07T07:45:01Z TYPE=ungated-health-backlog BAD=0 WARN=1 MODE=POST_PVE03_ROOT_CLEANUP_TRIAGE TOTAL=8 SUPERSEDED=4 STALE=2 POLICY_GAP_NEXT_STAGE=1 ACTIONABLE_CAPACITY=1 PVE03_ROOT_USAGE_PERCENT=31 PVE03_STAGING_USAGE_PERCENT=83 REPORT=/var/lib/homelab-ungated-health-backlog/latest.md
151-1190:/var/lib/homelab-health/mail-cloud-restore-drill.txt:1:STATUS=STALE
152-1191:/var/lib/homelab-health/mail-cloud-restore-drill.txt:20:FRESHNESS=STALE
153-1192:/var/lib/homelab-health/edge-restore-proofs-offhost.txt:1:STATUS=OK TS=20260624T190040Z TYPE=edge-restore-proofs-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge ITEMS=14 OK=14 ERROR=0 BYTES=226024287 SECRET_PRINTED=NO
--
156-1207:/var/lib/homelab-health/filebrowser-restore.txt:1:STATUS=STALE
157-1208:/var/lib/homelab-health/filebrowser-restore.txt:20:FRESHNESS=STALE
158:1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed
159-1494:2026-07-21T08:03:07Z	GITEA-PUSH-AUTH-001	fcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94	1	FAIL	gitea-runtime
160-1495:2026-07-21T08:13:32Z	GITEA-TOKEN-CLI-HELP-003	f1d0f967eed6490ad822944a42108c82dc4b59c9fd2557d8c64624dcb6853cd4	1	FAIL	gitea-runtime
161-1496:2026-07-21T08:14:52Z	GITEA-CLI-CAPABILITIES-004	2cd6c4d28c8f646d8063105fca80b2f80d38c6a5db3a3eb76869de326b12d2d6	1	FAIL	gitea-runtime
162-1497:2026-07-21T09:49:51Z	RUNTIME-RECEIVER-STATE-027	0831002c78c26f35826cd7199995e335ceadcd39f08b9bac23882a0fbb59c8cc	127	FAIL	gitea-runtime
163-1498:2026-07-21T10:26:18Z	RUNTIME-RECEIVER-CLEANUP-036	65fbcfbce38a55723a92a8790429d6567389321f030f02cb4bf301cbe67e2531	1	FAIL	gitea-runtime
164:1499:2026-07-21T11:35:23Z	BACKUP-DUAL-CLOUD-PREFLIGHT-059	c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479	1	FAIL	backup-audit
165:1500:2026-07-21T12:28:26Z	BACKUP-VM-DUMP-PIPELINE-073	cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b	1	FAIL	backup-audit
166:1501:2026-07-21T12:42:53Z	BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075	a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936	143	FAIL	backup-audit
167:1502:2026-07-21T12:56:52Z	BACKUP-DRAIN-LOCAL-ARCHIVES-079	531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32	75	FAIL	backup-remediation
168:1503:2026-07-21T13:04:48Z	BACKUP-FINAL-PROOF-082	9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601	127	FAIL	backup-audit
169-1504:2026-07-21T13:07:09Z	CLUSTER-CLEANUP-INVENTORY-083	09d3f14ca0205936a910dc7af7991474030273a47d76fa033ea6a3185c083922	1	FAIL	cluster-cleanup
170:1505:2026-07-21T13:14:27Z	CLUSTER-BACKUP-POSTCLEAN-PROOF-086	601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da	127	FAIL	backup-cleanup
171:1506:2026-07-21T13:15:21Z	BACKUP-POLICY-CLOSEOUT-087	d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52	1	FAIL	backup-cleanup
172:1507:2026-07-21T13:17:20Z	BACKUP-CLEANUP-FINAL-CLOSEOUT-088	5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6	1	FAIL	backup-cleanup
173-=== ACTIONABLE_STORAGE ===
174-40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.
175-454:local             dir     active       102626232        41916788        55450180   40.84%
176-=== OPEN_ERROR_REGISTER ===
=== OPEN_ERROR_REGISTER ===
=== OPEN_ERROR_REGISTER ===
NONE
=== RECENT_FAILED_COMMANDS_NORMALIZED ===
CRITICAL_FIRST_ACTION=COMPLETE

OUTPUT_END
CHAT_OUTPUT_END
