From 945cefe3e1bbec2f06e001aeef05fec40f6ce783 Mon Sep 17 00:00:00 2001 From: homelab-runtime-publisher Date: Tue, 21 Jul 2026 14:44:43 +0000 Subject: [PATCH] runtime: publish CRITICAL-DIAGNOSTIC-DECISION-104 --- .../CRITICAL-DIAGNOSTIC-DECISION-104.json | 30 + .../CRITICAL-DIAGNOSTIC-DECISION-104.txt | 41 + runtime/latest.json | 16 +- runtime/latest.txt | 2334 +---------------- 4 files changed, 96 insertions(+), 2325 deletions(-) create mode 100644 runtime/history/CRITICAL-DIAGNOSTIC-DECISION-104.json create mode 100644 runtime/history/CRITICAL-DIAGNOSTIC-DECISION-104.txt diff --git a/runtime/history/CRITICAL-DIAGNOSTIC-DECISION-104.json b/runtime/history/CRITICAL-DIAGNOSTIC-DECISION-104.json new file mode 100644 index 0000000..fc1ab05 --- /dev/null +++ b/runtime/history/CRITICAL-DIAGNOSTIC-DECISION-104.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "CRITICAL-DIAGNOSTIC-DECISION-104", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-21T14:44:42Z", + "finished_at_utc": "2026-07-21T14:44:43Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "2d76a56126624be128d7ea0cc6526f032944f0e28b5fa7afa7f4a3a763b60835", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "8b71df5e3fea77b866119cdf7ce9025e569b8bb49ca9484943ad44465f53c5dc", + "sanitized_output_sha256": "8b71df5e3fea77b866119cdf7ce9025e569b8bb49ca9484943ad44465f53c5dc", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "=== IMMUTABLE_RESULT_VERIFIED ===\nCOMMAND_ID=CRITICAL-FIRST-DIAGNOSTIC-103\nIMMUTABLE_COMMIT=c4c3e49dae61e1904c196db23d9095d8f35f620c\ncut: '\"': No such file or directory\nJSON_SHA256=\ncut: '\"': No such file or directory\nTXT_SHA256=\n=== DECISION_COUNTS ===\nQUORUM_RISKS=0\nBACKUP_RISKS=1202\nKERNEL_RISKS=0\nFAILED_UNIT_RISKS=6\nCAPACITY_RISKS=0\n=== NEXT_ACTION ===\nPRIORITY=P0_BACKUP_POLICY\nEVIDENCE=● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nCRITICAL_DIAGNOSTIC_DECISION=COMPLETE\n" +} diff --git a/runtime/history/CRITICAL-DIAGNOSTIC-DECISION-104.txt b/runtime/history/CRITICAL-DIAGNOSTIC-DECISION-104.txt new file mode 100644 index 0000000..f579bfd --- /dev/null +++ b/runtime/history/CRITICAL-DIAGNOSTIC-DECISION-104.txt @@ -0,0 +1,41 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=CRITICAL-DIAGNOSTIC-DECISION-104 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=2d76a56126624be128d7ea0cc6526f032944f0e28b5fa7afa7f4a3a763b60835 +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=8b71df5e3fea77b866119cdf7ce9025e569b8bb49ca9484943ad44465f53c5dc +SANITIZED_OUTPUT_SHA256=8b71df5e3fea77b866119cdf7ce9025e569b8bb49ca9484943ad44465f53c5dc +OUTPUT_BEGIN +=== IMMUTABLE_RESULT_VERIFIED === +COMMAND_ID=CRITICAL-FIRST-DIAGNOSTIC-103 +IMMUTABLE_COMMIT=c4c3e49dae61e1904c196db23d9095d8f35f620c +cut: '"': No such file or directory +JSON_SHA256= +cut: '"': No such file or directory +TXT_SHA256= +=== DECISION_COUNTS === +QUORUM_RISKS=0 +BACKUP_RISKS=1202 +KERNEL_RISKS=0 +FAILED_UNIT_RISKS=6 +CAPACITY_RISKS=0 +=== NEXT_ACTION === +PRIORITY=P0_BACKUP_POLICY +EVIDENCE=● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state +CRITICAL_DIAGNOSTIC_DECISION=COMPLETE + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index 9befe55..fc1ab05 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "CRITICAL-FIRST-DIAGNOSTIC-103", + "command_id": "CRITICAL-DIAGNOSTIC-DECISION-104", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", "component": "cluster-audit", - "started_at_utc": "2026-07-21T14:39:59Z", - "finished_at_utc": "2026-07-21T14:40:02Z", + "started_at_utc": "2026-07-21T14:44:42Z", + "finished_at_utc": "2026-07-21T14:44:43Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "32ad0722a84aebeb4f1e33dab07a771683458bec2422895b557411be769cd3f2", + "command_sha256": "2d76a56126624be128d7ea0cc6526f032944f0e28b5fa7afa7f4a3a763b60835", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "b23c2f575c56d419a6dfb673cbf42ce15bc9a9c8e2892d3d6a6f9a91c9a7a3ce", - "sanitized_output_sha256": "2a82a08aa78353751e68781b74d3029143cd010fb8931ff0b43efc0b8686b853", - "output_truncated_in_json": true, + "raw_evidence_sha256": "8b71df5e3fea77b866119cdf7ce9025e569b8bb49ca9484943ad44465f53c5dc", + "sanitized_output_sha256": "8b71df5e3fea77b866119cdf7ce9025e569b8bb49ca9484943ad44465f53c5dc", + "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "=== VERIFIED_PREVIOUS_RESULT ===\nCOMMAND_ID=CRITICAL-FIRST-ACTION-102\nIMMUTABLE_COMMIT=4cf04bbfdae66e66e09e451a6a5eae4abb874704\ncut: '\"': No such file or directory\nJSON_SHA256=\ncut: '\"': No such file or directory\nTXT_SHA256=\nPREVIOUS_COMMAND_IN_ERROR_REGISTER=no\n=== FIRST_PRIORITY_FINDINGS ===\n=== FINDING_COUNTS ===\nP0_MATCHES=45\nP1_MATCHES=87\nSTORAGE_RISK_MATCHES=2\nRECENT_FAILED_COMMAND_LINES=0\nBOOTSTRAP_SECTION_LINES=0\n=== ACTIONABLE_P0 ===\n=== ACTIONABLE_P0 ===\n86:398:- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained.\n155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.\n203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.\n486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800\n508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000\n513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800\n546:● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\n548:● homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\n1102:/var/lib/homelab-health/backup-matrix-cluster-final-seal.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z\n1104:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-production-closure.txt:1:STATUS=OK TS=2026-07-13T08:12:14Z TYPE=crypto-portfolio-ui-center-alert-currency-production-closure BAD=0 PRODUCTION_CHANGED=YES_UI_ONLY INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 POSTCHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION_ROLLBACK_COUNT=3 RETENTION_OPTIONAL_OTHER_CANDIDATES=1 RETENTION_REMOVED_ROLLBACK_COUNT=0 DANGLING_IMAGES_REMAINING=2 UNREFERENCED_DANGLING_IMAGES=0 DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE_CLEAN=YES PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES READY_FOR_UI_FINAL_SEAL=YES\n1106:/var/lib/homelab-health/cluster-admin-stage3-postapply-backup.txt:1:STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z\n1109:/var/lib/homelab-health/pve01-core-monitoring-app-restorecheck.txt:1:STATUS=OK TS=2026-07-08T15:57:27Z TYPE=pve01-core-monitoring-app-restorecheck BAD=0 MODE=APP_LEVEL_RESTORE_CHECKS_TWO_CLOUDS FULL_VM_RESTORE_AVAILABLE=NO TOTAL_APPS=30 OK_APPS=30 FAIL_LIST=none REPORT=/var/lib/homelab-node-loss/pve01-core-monitoring-app-level/app-restorecheck-latest.md\n1118:/var/lib/homelab-health/hapusya-site-final-seal.txt:1:STATUS=OK TS=2026-07-13T12:33:08Z TYPE=hapusya-site-final-seal BAD=0 APP=hapusya-site URL=https://hapusya.ru/ PRESEAL_DESIRED_STATE_HEAD=69966eb83fc065141df3c46df0d6c0f282275046 ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb APPLY_MANIFEST_SHA256=f44992e3e0cc443b372779cb35d9e899d904a9960a9de03c14559886fdb1fc75 PRODUCTION_BROWSER_MANIFEST_SHA256=9f3cdf3063f8724d1d1c1ad619f47d6f98d434c3422a74903585146858a8dfac BROWSER_SMOKE=OK BROWSER_RUNS=5 BROWSER_ERRORS=0 DESKTOP_VIDEO=OK MOBILE_VIDEO=OK STATIC_FALLBACK=OK BACKUP_JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 TWO_CLOUDS=OK RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK DESIRED_STATE_SYNC=OK GENERATOR_SHA256=507bbbc2691f1f08cb1dd3660ed76cf13a8c36c23833d5fdec40cf6e07486cb8 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 POLICY_SHA256=4bf996ff451ae3ec56eb2fcd42842606cd97295b4d5141aedb135c956395e4b5 EVIDENCE_ROOT=/var/lib/homelab-change-sets/hapusya-site-final-seal/20260713T123308Z EVIDENCE_MANIFEST_SHA256=7eb993a17ac18a1f7c9ae0b52d745d3778833d15626ed34c82496f74916eb26a PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY DESIRED_STATE_CHANGED=YES_FINAL_SEAL_COMMIT_ONLY ALL_HAPUSYA_SITE_TAILS_CLOSED=YES TASK_COMPLETE=YES\n1120:/var/lib/homelab-health/pve03-failed-unit-cleanup-closure.txt:1:STATUS=OK TS=2026-07-07T12:52:21Z TYPE=pve03-failed-unit-cleanup-closure BAD=0 WARN=0 MODE=READ_ONLY_CLOSURE LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CP_MATCH=YES RESTORE_CHECK=YES PVE03_FAILED_UNITS=0 REPORT=/var/lib/homelab-pve03-failed-unit-cleanup/closure-latest.md\n1128:/var/lib/homelab-health/cluster-admin-backup-webpanel-final-seal.txt:1:STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[PRIVATE_IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md\n1132:/var/lib/homelab-health/appbackupctl-routing.txt:1:STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z\n1134:/var/lib/homelab-health/cluster-admin-stage3-api-apply.txt:1:STATUS=OK TS=2026-07-13T03:22:35Z TYPE=cluster-admin-stage3-api-apply BAD=0 COMMAND_PHYSICAL_LINES=1 RESULT=APPLIED PRODUCTION_CHANGED=LIVE_APP_AND_DESIRED_STATE PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=YES CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_PREAPPLY_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES PREAPPLY_BACKUP_REUSED=YES AST_IDENTICAL_NORMALIZATION=YES EXISTING_ROUTES_BASELINE=OK NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES OPENAPI_DOCS_EXPOSURE_CHANGED=NO CORE_DATABASE_STATE=1|1|9|7|1 INCIDENT_ENGINE=ACTIVE VM180_FAILED_UNITS=0 ROLLBACK_CHECKPOINT=VERIFIED OLD_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a NEW_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 READY_FOR_STAGE3_AUTHENTICATED_PRODUCTION_VERIFY=YES STAGE3_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-production-apply/20260713T032235Z\n1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL\n1146:/var/lib/homelab-health/special-vm-emergency-non-app-final-seal.txt:1:STATUS=OK TS=2026-07-14T01:15:48Z TYPE=special-vm-emergency-non-app-final-seal BAD=0 WARN=0 FAMILY_COVERAGE=11/11 MATRIX_STATUS=OK JOBS_TOTAL=40 JOBS_OK=36 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 COMPATIBILITY_OUTPUTS=31 REQUIRED_GREEN_JOBS=14 VM_FULL=OK VM_LOCAL_DUMPS=OK EDGE_VM_SOURCE=OK EDGE_VM_CLOUD=OK NEXTCLOUD_VM_SOURCE=homelab_vm_local_dumps_2cloud NEXTCLOUD_VM_CLOUD=OK EMERGENCY_PACK=OK SKLADCHIK_REPORTS_MONITOR=OK GOLDEN_STATE=OK ROUTER_RUNNING_CONFIG=OK MAIL_CLOUD_CRITICAL=OK MAIL_CLOUD_EDGE=OK MAIL_CLOUD_NEXTCLOUD=OK WRITE_MODE=legacy READ_MODE=dual MIGRATE_EXISTING=no SOURCE_BACKUP_RUN=NO CLOUD_OPERATIONS_RUN=NO RETENTION_RUN=NO CLOUD_PURGE_RUN=NO CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=b26f498b55de5d80bcbb4b9fb3a9bccded9f99b4 CURRENT_SPECIAL_VM_EMERGENCY_NON_APP_STAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/special-vm-emergency-non-app-final-seal/20260714T011514Z\n1151:/var/lib/homelab-health/crypto-portfolio-ui-preapply-backup.txt:1:STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z\n1157:/var/lib/homelab-health/legacy-monitoring-backups-migrated.txt:1:STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK\n1159:/var/lib/homelab-health/cluster-admin-stage2-final-seal.txt:1:STATUS=OK TS=2026-07-13T01:45:29Z TYPE=cluster-admin-stage2-final-seal BAD=0 STAGE1_CLOSED=YES STAGE2_APPLY=OK PRODUCTION_CHANGED=BACKUP_HEALTH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_SCHEMA_CHANGED=NO_ADDITIONAL_CHANGE_DURING_SEAL INCIDENT_ENGINE_RESTARTED=NO MIGRATION_ID=002_canonical_foundation_sync FUNCTION=cluster_admin_sync_foundation COMMITTED_SYNC=YES MULTICYCLE_COLLECTOR_TEST=OK MULTICYCLE_NEW_RUNS=2 MULTICYCLE_GOOD_RUNS=2 ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK RETIRED_PLACEHOLDERS_ALLOWED=YES FOUNDATION_COUNTS=85|21|2|229|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0|0 ORPHANS=0 POSTAPPLY_DIRECT_SNAPSHOT=VERIFIED POSTAPPLY_RESTORE_TEST=OK POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T014757Z LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE2_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage2/stage2-final-seal-latest.txt REPORT_SHA256=32d4d37d82aa25194d789083d0e7ad2e9052ee38a72df5312b711226898cf1a6 BLOCKERS=/var/lib/homelab-cluster-admin-stage2/20260713T014529Z/blockers.txt\n1169:/var/lib/homelab-health/cluster-admin-stage3-final-seal.txt:1:STATUS=OK TS=2026-07-13T03:46:30Z TYPE=cluster-admin-stage3-final-seal BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=HEALTH_REFRESH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO STAGE2_CLOSED=YES STAGE3_APPLY=OK AUTHENTICATED_PRODUCTION_VERIFY=OK AUTHENTICATED_PRODUCTION_TESTS=25 APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 NEW_GET_ROUTES=7 DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY OPENAPI_DOCS_EXPOSURE=PRESERVED_DISABLED POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z POSTAPPLY_APPBACKUP_RESTORE_CHECK=OK COLLECTOR_STABILITY=OK LATEST_COLLECTOR_RUN_ID=3114 CANONICAL_CONSISTENCY=0|0|0|0|0 OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE3_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage3/stage3-final-seal-latest.txt REPORT_SHA256=6f67545cd7f0a75b87146c1df74c61c3b88e142f9e53a96b083ec27209e6d265\n1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL\n1172:/var/lib/homelab-health/backup-matrix-desired-state-persistence.txt:1:STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z\n1175:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES\n1184:/var/lib/homelab-health/cluster-admin-stage4d-adapter-candidates-in-isolation.txt:1:STATUS=OK TS=2026-07-14T06:14:58Z TYPE=cluster-admin-stage4d-adapter-candidates-in-isolation-final-seal BAD=0 STAGE4D_CLOSED=YES NEXT_STAGE=4E_RESTRICTED_PROBE_AGENT_DESIGN PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO APPLICATION_CHANGED=NO ENVELOPE_TOTAL=48 PROMETHEUS_ENVELOPES=5 BACKUP_MATRIX_ENVELOPES=40 HEALTH_ENVELOPES=3 LOCAL_REMOTE_OUTPUT_MATCH=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=e77fb37a0fa0033fe46558045e31dd92b6d1238a DESIRED_STATE_CLEAN=YES PVE01_FAILED_UNITS=0 REPORT=/var/lib/homelab-cluster-admin-stage4/20260714T061458Z/stage4d-adapter-candidates-in-isolation-seal.txt REPORT_SHA256=da6244c0a8a458b508fd58a0fec9cb1e6fa8da98a36f44d1368cbe947f4b5835\n1185:/var/lib/homelab-health/backup-matrix-cluster-desired-state-sync.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z\n1187:/var/lib/homelab-health/cluster-admin-stage3-api-runtime-dry-run.txt:1:STATUS=OK TS=2026-07-13T02:53:34Z TYPE=cluster-admin-stage3-api-runtime-dry-run BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=TEMPORARY_ISOLATED_RUNTIME_AND_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO PRODUCTION_SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ISOLATED_BIND=127.0.0.1:18081 TEMPORARY_DATABASE_RESTORED=YES TEMPORARY_DATABASE_DROPPED=YES TEMPORARY_DATABASE_UNCHANGED=YES RUNTIME_TESTS=41 RUNTIME_FAILED_TESTS=0 EXISTING_ROUTES_BACKWARD_COMPATIBLE=YES NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES AUTHENTICATED_NEW_ROUTES=OK PAGINATION=OK FILTER_VALIDATION=OK ENTITY_DETAIL=OK ENTITY_DEPENDENCIES=OK SOURCES=OK POLICIES=OK COLLECTOR_RUNS=OK DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY FORBIDDEN_RESPONSE_FIELDS=ABSENT OPENAPI_DOCS_EXPOSURE_CHANGED=NO SECURITY_HEADERS=OK ISOLATED_RUNTIME_STOPPED=YES ISOLATED_PORT_CLOSED=YES LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK READY_FOR_STAGE3_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z RESULT=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z/runtime-result.json\n1188:/var/lib/homelab-health/crypto-portfolio-remove-grt-failed-units-diagnostic.txt:1:STATUS=OK TS=2026-07-13T05:41:00Z TYPE=crypto-portfolio-remove-grt-failed-units-diagnostic BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=NO_DIAGNOSTIC_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO FAILED_STATE_RESET=NO CLOUD_WRITE_RUN=NO PVE01_FAILED_UNIT_COUNT=2 PVE01_FAILED_UNIT_NAMES=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service CRYPTO_PORTFOLIO_FAILED_UNIT_COUNT=0 OTHER_FAILED_UNIT_COUNT=2 CURRENT_CRYPTO_PRECHANGE_HEALTH=YES UNIT_DETAILS_CAPTURED=YES READY_FOR_TARGETED_FAILED_UNIT_DECISION=YES DIAGNOSTIC_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-failed-units-diagnostic/20260713T054100Z\n1189:/var/lib/homelab-health/crypto-portfolio-remove-grt-production-apply.txt:1:STATUS=OK TS=2026-07-13T05:47:54Z TYPE=crypto-portfolio-remove-grt-production-apply BAD=0 COMMAND_PHYSICAL_LINES=1 APPLY_REVISION=V3 FIX_UID_READONLY=YES FIX_READINESS_ENDPOINT=YES FIX_ROLLBACK_DATABASE_METADATA=YES PRODUCTION_CHANGED=YES_REMOVE_GRT DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND PRODUCTION_CONTAINER_RECREATED=YES CLOUD_WRITE_RUN=NO ROLLBACK_ON_FAILURE=ARMED_NOT_NEEDED TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 PRECHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CHART_HEALTH_SCRIPT_SHA256=c16473f16390eeeff4d4eb701db604093564dd78d0be2d30b4cc8ffeeb22364a ACTIVE_COINS=HBAR,POL GRT_SOURCE_REFERENCES=0 GRT_DATABASE_ROWS=0|0|0 GRT_MARKET_ENTRY=ABSENT GRT_CACHE_ENTRIES=ABSENT DATABASE_MODE=0644 DATABASE_UID=10001 DATABASE_GID=10001 DATABASE_INTEGRITY=ok WORKER_STATUS=OK WORKER_COIN_COUNT=2 PUBLIC_CODES=200|303|401|200|200|401 CHART_HEALTH=OK CHART_SYMBOLS=HBAR,POL CHART_HEALTH_TIMER=ACTIVE VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_UNITS=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES REMOTE_STAGE_CLEANED=YES READY_FOR_GRT_POSTAPPLY_VERIFY_BACKUP_SYNC=YES APPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-production-apply-v3/20260713T054754Z\n1193:/var/lib/homelab-health/crypto-portfolio-runtime-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES\n1194:/var/lib/homelab-health/backup-matrix-cluster-preseal-convergence.txt:1:STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z\n1202:/var/lib/homelab-health/node-loss-final-closure.txt:1:STATUS=OK TS=2026-07-08T16:35:56Z TYPE=node-loss-final-closure BAD=0 MODE=ALL_NODE_LOSS_DRILLS_SEALED PVE03_EDGE_RESTORE=OK PVE03_EDGE_FRESHNESS=OK PVE02_FORUM_RUNTIME=OK PVE01_NEXTCLOUD_DNS=OK PVE01_CORE_MONITORING_APPLEVEL=OK FULL_VM_GAP_VM170_VM171=ACKNOWLEDGED_APP_LEVEL_ONLY TEST_VMS_CLEANED=YES DNS_PAIR_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-node-loss/final-closure-latest.md\n1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed\n1332:/var/lib/homelab-health/codevipe-cloud-backup-reset.txt:1:STATUS=OK TS=2026-07-08T05:25:22Z TYPE=remote-failed-unit-codevipe-reset BAD=0 WARN=0 MODE=RESET_STALE_FAILED_STATE UNIT=codevipe-cloud-backup.service REASON=last_journal_success_and_no_timer ACTION=reset_failed_only\n1426:● homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image\n1499:2026-07-21T11:35:23Z\tBACKUP-DUAL-CLOUD-PREFLIGHT-059\tc6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479\t1\tFAIL\tbackup-audit\n1500:2026-07-21T12:28:26Z\tBACKUP-VM-DUMP-PIPELINE-073\tcd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b\t1\tFAIL\tbackup-audit\n1501:2026-07-21T12:42:53Z\tBACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075\ta9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936\t143\tFAIL\tbackup-audit\n1502:2026-07-21T12:56:52Z\tBACKUP-DRAIN-LOCAL-ARCHIVES-079\t531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32\t75\tFAIL\tbackup-remediation\n1503:2026-07-21T13:04:48Z\tBACKUP-FINAL-PROOF-082\t9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601\t127\tFAIL\tbackup-audit\n1505:2026-07-21T13:14:27Z\tCLUSTER-BACKUP-POSTCLEAN-PROOF-086\t601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da\t127\tFAIL\tbackup-cleanup\n1506:2026-07-21T13:15:21Z\tBACKUP-POLICY-CLOSEOUT-087\td79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52\t1\tFAIL\tbackup-cleanup\n1507:2026-07-21T13:17:20Z\tBACKUP-CLEANUP-FINAL-CLOSEOUT-088\t5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6\t1\tFAIL\tbackup-cleanup\n=== ACTIONABLE_P1 ===\n=== ACTIONABLE_P1 ===\n=== ACTIONABLE_P1 ===\n26:1:# HOMELAB ASSISTANT ERROR REGISTER\n28:77:21. Ошибка: nested PHP php -r дал Parse error на forum-prod.\n29:83:Команда 108 дала PHP Parse error на smtpHost и пустой proof-файл.\n30:88:Команда 111 не изменила smtpSsl: OLD_SMTPSSL=false, SQL ERROR 1064, NEW_SMTPSSL=false.\n40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.\n41:154:37. Coverage gap: pve03_staging missing from disk-space health coverage.\n42:159:Факт: terminal output around command 222 showed paste artifact near \"FAIL; fi\".\n45:171:Факт: first reconciliation wrote STATUS=ERROR because router-running-config-mail-cloud.txt is on pve01, not edge-vm.\n51:194:- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200.\n52:195:- If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed.\n55:199:- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('.\n60:227:- Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli].\n79:368:- Evidence: proof 542 showed all five zones missing and DNS record create probes failed.\n83:388:- Impact: web routing is OK, but mail-related DNS may still contain stale provider data.\n84:392:- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt.\n85:397:- Context: SPF cleanup command 565 attempted to replace stale SPF records.\n93:416:- Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line.\n95:429:- Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth.\n97:435:- Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory contains old codevipe-public/codevipe.sql files, not current five-forum archive.\n120:621:- Certbot on edge rejected --dns-cloudflare-* arguments: dns-cloudflare plugin missing/not loaded.\n129:657:- Факт: bash начал интерпретировать Python-код, появились syntax error и вторичный prompt >.\n131:675:- When asked to fix one missing Uptime Kuma service, add only one monitor, then generate proposals separately.\n134:681:- Dockge inactive items after migration can be stale compose folders, not stopped containers.\n135:684:- Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions.\n145:737:- The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults.\n149:761:- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults.\n153:774:- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects.\n155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.\n171:839:- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`.\n180:865:- Symptom: error-register candidate construction stopped with RC1 before applying the candidate.\n183:876:- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis.\n201:948:BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative.\n203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.\n220:- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR.\n235:- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring.\n236:- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification.\n245:- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first.\n250:- Stale moved edge stack folders archived under /opt/dockge-stale-archive/20260702T230442Z on edge-vm; no runtime containers were deleted.\n252:- Final target: remote not_visible count must be 0 and edge stale remaining matches must be 0.\n258:- Stale moved stacks were archived, not deleted.\n316:- Фактические поля collector_runs: finished_at и error_text; completed_at и error отсутствуют.\n320:- Production осталась 0|0|OK, timer активен, failed units отсутствуют.\n372: \"missing\": [\n459:STATUS=FAIL\n475:JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000\n486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800\n495:JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000\n508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000\n510:JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000\n513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800\n676:2026-07-08T09:26:36.7403618890 390 /var/lib/homelab-health/dockge-stale-stack-classification-after-agents.txt\n681:2026-07-08T09:26:36.7406026680 390 /var/lib/homelab-health/homepage-missing-card-proposals.txt\n684:2026-07-08T09:26:36.7408259230 390 /var/lib/homelab-health/homepage-missing-card-proposals-final.txt\n689:2026-07-08T09:26:36.7410552890 389 /var/lib/homelab-health/dockge-stale-archive-last.txt\n1114:/var/lib/homelab-health/service-registry.txt:1:STATUS=OK TS=2026-07-21T13:15:17Z TYPE=service-registry SERVICES=12 MISSING=0 REGISTRY=/var/lib/homelab-registry/service-registry-latest.tsv\n1122:/var/lib/homelab-health/nextcloud-restore-proof-offhost.txt:1:STATUS=OK TS=20260624T190330Z TYPE=nextcloud-restore-proof-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge/nextcloud-aio/nextcloud-aio-restore-proof/20260624T120344Z ITEMS=1 OK=1 ERROR=0 BYTES=2135684 FILES=7 SECRET_PRINTED=NO\n1127:/var/lib/homelab-health/crypto-portfolio-remove-grt-recovery-audit.txt:1:STATUS=ERROR TS=2026-07-13T05:39:07Z TYPE=crypto-portfolio-remove-grt-recovery-audit BAD=1 COMMAND_PHYSICAL_LINES=1 ROOT_CAUSE=READINESS_ENDPOINT_MISMATCH RECOVERY_VERIFIED=NO TASK_COMPLETE=NO READY_FOR_GRT_APPLY_SCRIPT_FIX=NO AUDIT_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-recovery-audit-v2/20260713T053907Z\n1133:/var/lib/homelab-health/final-readiness.txt:1:STATUS=WARN TS=2026-07-21T05:31:30Z TYPE=final-readiness BAD=5 PASS=48 FAIL=5 REPORT=/var/lib/homelab-final-readiness/latest.md\n1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL\n1152:/var/lib/homelab-health/router-hard-power.txt:1:STATUS=ERROR TS=2026-07-16T06:20:34Z MODE=cycle REMOTE=[PRIVATE_IP] RC=74\n1162:/var/lib/homelab-health/private-vpn-hosts.txt.bak-ok-wrapper-20260624T132705Z:16:STATUS=ERROR\n1164:/var/lib/homelab-health/paperless-restore.txt:1:STATUS=STALE\n1165:/var/lib/homelab-health/paperless-restore.txt:20:FRESHNESS=STALE\n1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL\n1174:/var/lib/homelab-health/ungated-health-backlog.txt:1:STATUS=OK TS=2026-07-07T07:45:01Z TYPE=ungated-health-backlog BAD=0 WARN=1 MODE=POST_PVE03_ROOT_CLEANUP_TRIAGE TOTAL=8 SUPERSEDED=4 STALE=2 POLICY_GAP_NEXT_STAGE=1 ACTIONABLE_CAPACITY=1 PVE03_ROOT_USAGE_PERCENT=31 PVE03_STAGING_USAGE_PERCENT=83 REPORT=/var/lib/homelab-ungated-health-backlog/latest.md\n1190:/var/lib/homelab-health/mail-cloud-restore-drill.txt:1:STATUS=STALE\n1191:/var/lib/homelab-health/mail-cloud-restore-drill.txt:20:FRESHNESS=STALE\n1192:/var/lib/homelab-health/edge-restore-proofs-offhost.txt:1:STATUS=OK TS=20260624T190040Z TYPE=edge-restore-proofs-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge ITEMS=14 OK=14 ERROR=0 BYTES=226024287 SECRET_PRINTED=NO\n1195:/var/lib/homelab-health/uptime-kuma-pending-health-monitors-fix.txt:8:NOTE=Paperless Restore may become red if its health file contains STATUS=ERROR; that is a real restore-health issue, not a Kuma pending issue.\n1199:/var/lib/homelab-health/cluster-admin-stage0-convergence.txt:1:STATUS=WARN TS=2026-07-12T23:12:59Z TYPE=cluster-admin-stage0-targeted-convergence BAD=2 PROFILE_OK=NO CLUSTER_PASSPORT=FAIL FINAL_READINESS=FAIL OVERALL=FAIL DUTY_ADMIN=FAIL FAILED_UNITS=0 LIVE_MATCH_DESIRED_STATE=YES READY_FOR_FINAL_SEAL=NO REPORT_DIR=/var/lib/homelab-cluster-admin-stage0/readiness-profile-20260712T231259Z PRODUCTION_CHANGED=HEALTH_REGENERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO\n1207:/var/lib/homelab-health/filebrowser-restore.txt:1:STATUS=STALE\n1208:/var/lib/homelab-health/filebrowser-restore.txt:20:FRESHNESS=STALE\n1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed\n1494:2026-07-21T08:03:07Z\tGITEA-PUSH-AUTH-001\tfcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94\t1\tFAIL\tgitea-runtime\n1495:2026-07-21T08:13:32Z\tGITEA-TOKEN-CLI-HELP-003\tf1d0f967eed6490ad822944a42108c82dc4b59c9fd2557d8c64624dcb6853cd4\t1\tFAIL\tgitea-runtime\n1496:2026-07-21T08:14:52Z\tGITEA-CLI-CAPABILITIES-004\t2cd6c4d28c8f646d8063105fca80b2f80d38c6a5db3a3eb76869de326b12d2d6\t1\tFAIL\tgitea-runtime\n1497:2026-07-21T09:49:51Z\tRUNTIME-RECEIVER-STATE-027\t0831002c78c26f35826cd7199995e335ceadcd39f08b9bac23882a0fbb59c8cc\t127\tFAIL\tgitea-runtime\n1498:2026-07-21T10:26:18Z\tRUNTIME-RECEIVER-CLEANUP-036\t65fbcfbce38a55723a92a8790429d6567389321f030f02cb4bf301cbe67e2531\t1\tFAIL\tgitea-runtime\n1499:2026-07-21T11:35:23Z\tBACKUP-DUAL-CLOUD-PREFLIGHT-059\tc6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479\t1\tFAIL\tbackup-audit\n1500:2026-07-21T12:28:26Z\tBACKUP-VM-DUMP-PIPELINE-073\tcd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b\t1\tFAIL\tbackup-audit\n1501:2026-07-21T12:42:53Z\tBACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075\ta9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936\t143\tFAIL\tbackup-audit\n1502:2026-07-21T12:56:52Z\tBACKUP-DRAIN-LOCAL-ARCHIVES-079\t531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32\t75\tFAIL\tbackup-remediation\n1503:2026-07-21T13:04:48Z\tBACKUP-FINAL-PROOF-082\t9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601\t127\tFAIL\tbackup-audit\n1504:2026-07-21T13:07:09Z\tCLUSTER-CLEANUP-INVENTORY-083\t09d3f14ca0205936a910dc7af7991474030273a47d76fa033ea6a3185c083922\t1\tFAIL\tcluster-cleanup\n1505:2026-07-21T13:14:27Z\tCLUSTER-BACKUP-POSTCLEAN-PROOF-086\t601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da\t127\tFAIL\tbackup-cleanup\n1506:2026-07-21T13:15:21Z\tBACKUP-POLICY-CLOSEOUT-087\td79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52\t1\tFAIL\tbackup-cleanup\n1507:2026-07-21T13:17:20Z\tBACKUP-CLEANUP-FINAL-CLOSEOUT-088\t5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6\t1\tFAIL\tbackup-cleanup\n=== ACTIONABLE_STORAGE ===\n=== ACTIONABLE_STORAGE ===\n=== ACTIONABLE_STORAGE ===\n40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.\n454:local dir active 102626232 41916788 55450180 40.84%\n=== OPEN_ERROR_REGISTER ===\n=== FIRST_PRIORITY_CANDIDATES ===\n38-BOOTSTRAP_SECTION_LINES=0\n39-=== ACTIONABLE_P0 ===\n40:86:398:- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained.\n41:155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.\n42:203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.\n43:486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800\n44:508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000\n45:513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800\n46:546:● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\n47:548:● homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\n48:1102:/var/lib/homelab-health/backup-matrix-cluster-final-seal.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z\n49:1104:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-production-closure.txt:1:STATUS=OK TS=2026-07-13T08:12:14Z TYPE=crypto-portfolio-ui-center-alert-currency-production-closure BAD=0 PRODUCTION_CHANGED=YES_UI_ONLY INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 POSTCHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION_ROLLBACK_COUNT=3 RETENTION_OPTIONAL_OTHER_CANDIDATES=1 RETENTION_REMOVED_ROLLBACK_COUNT=0 DANGLING_IMAGES_REMAINING=2 UNREFERENCED_DANGLING_IMAGES=0 DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE_CLEAN=YES PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES READY_FOR_UI_FINAL_SEAL=YES\n50:1106:/var/lib/homelab-health/cluster-admin-stage3-postapply-backup.txt:1:STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z\n51:1109:/var/lib/homelab-health/pve01-core-monitoring-app-restorecheck.txt:1:STATUS=OK TS=2026-07-08T15:57:27Z TYPE=pve01-core-monitoring-app-restorecheck BAD=0 MODE=APP_LEVEL_RESTORE_CHECKS_TWO_CLOUDS FULL_VM_RESTORE_AVAILABLE=NO TOTAL_APPS=30 OK_APPS=30 FAIL_LIST=none REPORT=/var/lib/homelab-node-loss/pve01-core-monitoring-app-level/app-restorecheck-latest.md\n52:1118:/var/lib/homelab-health/hapusya-site-final-seal.txt:1:STATUS=OK TS=2026-07-13T12:33:08Z TYPE=hapusya-site-final-seal BAD=0 APP=hapusya-site URL=https://hapusya.ru/ PRESEAL_DESIRED_STATE_HEAD=69966eb83fc065141df3c46df0d6c0f282275046 ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb APPLY_MANIFEST_SHA256=f44992e3e0cc443b372779cb35d9e899d904a9960a9de03c14559886fdb1fc75 PRODUCTION_BROWSER_MANIFEST_SHA256=9f3cdf3063f8724d1d1c1ad619f47d6f98d434c3422a74903585146858a8dfac BROWSER_SMOKE=OK BROWSER_RUNS=5 BROWSER_ERRORS=0 DESKTOP_VIDEO=OK MOBILE_VIDEO=OK STATIC_FALLBACK=OK BACKUP_JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 TWO_CLOUDS=OK RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK DESIRED_STATE_SYNC=OK GENERATOR_SHA256=507bbbc2691f1f08cb1dd3660ed76cf13a8c36c23833d5fdec40cf6e07486cb8 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 POLICY_SHA256=4bf996ff451ae3ec56eb2fcd42842606cd97295b4d5141aedb135c956395e4b5 EVIDENCE_ROOT=/var/lib/homelab-change-sets/hapusya-site-final-seal/20260713T123308Z EVIDENCE_MANIFEST_SHA256=7eb993a17ac18a1f7c9ae0b52d745d3778833d15626ed34c82496f74916eb26a PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY DESIRED_STATE_CHANGED=YES_FINAL_SEAL_COMMIT_ONLY ALL_HAPUSYA_SITE_TAILS_CLOSED=YES TASK_COMPLETE=YES\n53:1120:/var/lib/homelab-health/pve03-failed-unit-cleanup-closure.txt:1:STATUS=OK TS=2026-07-07T12:52:21Z TYPE=pve03-failed-unit-cleanup-closure BAD=0 WARN=0 MODE=READ_ONLY_CLOSURE LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CP_MATCH=YES RESTORE_CHECK=YES PVE03_FAILED_UNITS=0 REPORT=/var/lib/homelab-pve03-failed-unit-cleanup/closure-latest.md\n54:1128:/var/lib/homelab-health/cluster-admin-backup-webpanel-final-seal.txt:1:STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[PRIVATE_IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md\n55:1132:/var/lib/homelab-health/appbackupctl-routing.txt:1:STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z\n56:1134:/var/lib/homelab-health/cluster-admin-stage3-api-apply.txt:1:STATUS=OK TS=2026-07-13T03:22:35Z TYPE=cluster-admin-stage3-api-apply BAD=0 COMMAND_PHYSICAL_LINES=1 RESULT=APPLIED PRODUCTION_CHANGED=LIVE_APP_AND_DESIRED_STATE PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=YES CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_PREAPPLY_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES PREAPPLY_BACKUP_REUSED=YES AST_IDENTICAL_NORMALIZATION=YES EXISTING_ROUTES_BASELINE=OK NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES OPENAPI_DOCS_EXPOSURE_CHANGED=NO CORE_DATABASE_STATE=1|1|9|7|1 INCIDENT_ENGINE=ACTIVE VM180_FAILED_UNITS=0 ROLLBACK_CHECKPOINT=VERIFIED OLD_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a NEW_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 READY_FOR_STAGE3_AUTHENTICATED_PRODUCTION_VERIFY=YES STAGE3_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-production-apply/20260713T032235Z\n57:1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL\n58:1146:/var/lib/homelab-health/special-vm-emergency-non-app-final-seal.txt:1:STATUS=OK TS=2026-07-14T01:15:48Z TYPE=special-vm-emergency-non-app-final-seal BAD=0 WARN=0 FAMILY_COVERAGE=11/11 MATRIX_STATUS=OK JOBS_TOTAL=40 JOBS_OK=36 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 COMPATIBILITY_OUTPUTS=31 REQUIRED_GREEN_JOBS=14 VM_FULL=OK VM_LOCAL_DUMPS=OK EDGE_VM_SOURCE=OK EDGE_VM_CLOUD=OK NEXTCLOUD_VM_SOURCE=homelab_vm_local_dumps_2cloud NEXTCLOUD_VM_CLOUD=OK EMERGENCY_PACK=OK SKLADCHIK_REPORTS_MONITOR=OK GOLDEN_STATE=OK ROUTER_RUNNING_CONFIG=OK MAIL_CLOUD_CRITICAL=OK MAIL_CLOUD_EDGE=OK MAIL_CLOUD_NEXTCLOUD=OK WRITE_MODE=legacy READ_MODE=dual MIGRATE_EXISTING=no SOURCE_BACKUP_RUN=NO CLOUD_OPERATIONS_RUN=NO RETENTION_RUN=NO CLOUD_PURGE_RUN=NO CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=b26f498b55de5d80bcbb4b9fb3a9bccded9f99b4 CURRENT_SPECIAL_VM_EMERGENCY_NON_APP_STAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/special-vm-emergency-non-app-final-seal/20260714T011514Z\n59:1151:/var/lib/homelab-health/crypto-portfolio-ui-preapply-backup.txt:1:STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z\n60:1157:/var/lib/homelab-health/legacy-monitoring-backups-migrated.txt:1:STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK\n61:1159:/var/lib/homelab-health/cluster-admin-stage2-final-seal.txt:1:STATUS=OK TS=2026-07-13T01:45:29Z TYPE=cluster-admin-stage2-final-seal BAD=0 STAGE1_CLOSED=YES STAGE2_APPLY=OK PRODUCTION_CHANGED=BACKUP_HEALTH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_SCHEMA_CHANGED=NO_ADDITIONAL_CHANGE_DURING_SEAL INCIDENT_ENGINE_RESTARTED=NO MIGRATION_ID=002_canonical_foundation_sync FUNCTION=cluster_admin_sync_foundation COMMITTED_SYNC=YES MULTICYCLE_COLLECTOR_TEST=OK MULTICYCLE_NEW_RUNS=2 MULTICYCLE_GOOD_RUNS=2 ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK RETIRED_PLACEHOLDERS_ALLOWED=YES FOUNDATION_COUNTS=85|21|2|229|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0|0 ORPHANS=0 POSTAPPLY_DIRECT_SNAPSHOT=VERIFIED POSTAPPLY_RESTORE_TEST=OK POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T014757Z LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE2_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage2/stage2-final-seal-latest.txt REPORT_SHA256=32d4d37d82aa25194d789083d0e7ad2e9052ee38a72df5312b711226898cf1a6 BLOCKERS=/var/lib/homelab-cluster-admin-stage2/20260713T014529Z/blockers.txt\n62:1169:/var/lib/homelab-health/cluster-admin-stage3-final-seal.txt:1:STATUS=OK TS=2026-07-13T03:46:30Z TYPE=cluster-admin-stage3-final-seal BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=HEALTH_REFRESH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO STAGE2_CLOSED=YES STAGE3_APPLY=OK AUTHENTICATED_PRODUCTION_VERIFY=OK AUTHENTICATED_PRODUCTION_TESTS=25 APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 NEW_GET_ROUTES=7 DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY OPENAPI_DOCS_EXPOSURE=PRESERVED_DISABLED POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z POSTAPPLY_APPBACKUP_RESTORE_CHECK=OK COLLECTOR_STABILITY=OK LATEST_COLLECTOR_RUN_ID=3114 CANONICAL_CONSISTENCY=0|0|0|0|0 OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE3_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage3/stage3-final-seal-latest.txt REPORT_SHA256=6f67545cd7f0a75b87146c1df74c61c3b88e142f9e53a96b083ec27209e6d265\n63:1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL\n64:1172:/var/lib/homelab-health/backup-matrix-desired-state-persistence.txt:1:STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z\n65:1175:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES\n66:1184:/var/lib/homelab-health/cluster-admin-stage4d-adapter-candidates-in-isolation.txt:1:STATUS=OK TS=2026-07-14T06:14:58Z TYPE=cluster-admin-stage4d-adapter-candidates-in-isolation-final-seal BAD=0 STAGE4D_CLOSED=YES NEXT_STAGE=4E_RESTRICTED_PROBE_AGENT_DESIGN PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO APPLICATION_CHANGED=NO ENVELOPE_TOTAL=48 PROMETHEUS_ENVELOPES=5 BACKUP_MATRIX_ENVELOPES=40 HEALTH_ENVELOPES=3 LOCAL_REMOTE_OUTPUT_MATCH=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=e77fb37a0fa0033fe46558045e31dd92b6d1238a DESIRED_STATE_CLEAN=YES PVE01_FAILED_UNITS=0 REPORT=/var/lib/homelab-cluster-admin-stage4/20260714T061458Z/stage4d-adapter-candidates-in-isolation-seal.txt REPORT_SHA256=da6244c0a8a458b508fd58a0fec9cb1e6fa8da98a36f44d1368cbe947f4b5835\n67:1185:/var/lib/homelab-health/backup-matrix-cluster-desired-state-sync.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z\n68:1187:/var/lib/homelab-health/cluster-admin-stage3-api-runtime-dry-run.txt:1:STATUS=OK TS=2026-07-13T02:53:34Z TYPE=cluster-admin-stage3-api-runtime-dry-run BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=TEMPORARY_ISOLATED_RUNTIME_AND_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO PRODUCTION_SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ISOLATED_BIND=127.0.0.1:18081 TEMPORARY_DATABASE_RESTORED=YES TEMPORARY_DATABASE_DROPPED=YES TEMPORARY_DATABASE_UNCHANGED=YES RUNTIME_TESTS=41 RUNTIME_FAILED_TESTS=0 EXISTING_ROUTES_BACKWARD_COMPATIBLE=YES NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES AUTHENTICATED_NEW_ROUTES=OK PAGINATION=OK FILTER_VALIDATION=OK ENTITY_DETAIL=OK ENTITY_DEPENDENCIES=OK SOURCES=OK POLICIES=OK COLLECTOR_RUNS=OK DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY FORBIDDEN_RESPONSE_FIELDS=ABSENT OPENAPI_DOCS_EXPOSURE_CHANGED=NO SECURITY_HEADERS=OK ISOLATED_RUNTIME_STOPPED=YES ISOLATED_PORT_CLOSED=YES LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK READY_FOR_STAGE3_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z RESULT=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z/runtime-result.json\n69:1188:/var/lib/homelab-health/crypto-portfolio-remove-grt-failed-units-diagnostic.txt:1:STATUS=OK TS=2026-07-13T05:41:00Z TYPE=crypto-portfolio-remove-grt-failed-units-diagnostic BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=NO_DIAGNOSTIC_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO FAILED_STATE_RESET=NO CLOUD_WRITE_RUN=NO PVE01_FAILED_UNIT_COUNT=2 PVE01_FAILED_UNIT_NAMES=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service CRYPTO_PORTFOLIO_FAILED_UNIT_COUNT=0 OTHER_FAILED_UNIT_COUNT=2 CURRENT_CRYPTO_PRECHANGE_HEALTH=YES UNIT_DETAILS_CAPTURED=YES READY_FOR_TARGETED_FAILED_UNIT_DECISION=YES DIAGNOSTIC_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-failed-units-diagnostic/20260713T054100Z\n70:1189:/var/lib/homelab-health/crypto-portfolio-remove-grt-production-apply.txt:1:STATUS=OK TS=2026-07-13T05:47:54Z TYPE=crypto-portfolio-remove-grt-production-apply BAD=0 COMMAND_PHYSICAL_LINES=1 APPLY_REVISION=V3 FIX_UID_READONLY=YES FIX_READINESS_ENDPOINT=YES FIX_ROLLBACK_DATABASE_METADATA=YES PRODUCTION_CHANGED=YES_REMOVE_GRT DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND PRODUCTION_CONTAINER_RECREATED=YES CLOUD_WRITE_RUN=NO ROLLBACK_ON_FAILURE=ARMED_NOT_NEEDED TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 PRECHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CHART_HEALTH_SCRIPT_SHA256=c16473f16390eeeff4d4eb701db604093564dd78d0be2d30b4cc8ffeeb22364a ACTIVE_COINS=HBAR,POL GRT_SOURCE_REFERENCES=0 GRT_DATABASE_ROWS=0|0|0 GRT_MARKET_ENTRY=ABSENT GRT_CACHE_ENTRIES=ABSENT DATABASE_MODE=0644 DATABASE_UID=10001 DATABASE_GID=10001 DATABASE_INTEGRITY=ok WORKER_STATUS=OK WORKER_COIN_COUNT=2 PUBLIC_CODES=200|303|401|200|200|401 CHART_HEALTH=OK CHART_SYMBOLS=HBAR,POL CHART_HEALTH_TIMER=ACTIVE VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_UNITS=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES REMOTE_STAGE_CLEANED=YES READY_FOR_GRT_POSTAPPLY_VERIFY_BACKUP_SYNC=YES APPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-production-apply-v3/20260713T054754Z\n71:1193:/var/lib/homelab-health/crypto-portfolio-runtime-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES\n72:1194:/var/lib/homelab-health/backup-matrix-cluster-preseal-convergence.txt:1:STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z\n73:1202:/var/lib/homelab-health/node-loss-final-closure.txt:1:STATUS=OK TS=2026-07-08T16:35:56Z TYPE=node-loss-final-closure BAD=0 MODE=ALL_NODE_LOSS_DRILLS_SEALED PVE03_EDGE_RESTORE=OK PVE03_EDGE_FRESHNESS=OK PVE02_FORUM_RUNTIME=OK PVE01_NEXTCLOUD_DNS=OK PVE01_CORE_MONITORING_APPLEVEL=OK FULL_VM_GAP_VM170_VM171=ACKNOWLEDGED_APP_LEVEL_ONLY TEST_VMS_CLEANED=YES DNS_PAIR_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-node-loss/final-closure-latest.md\n74:1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed\n75:1332:/var/lib/homelab-health/codevipe-cloud-backup-reset.txt:1:STATUS=OK TS=2026-07-08T05:25:22Z TYPE=remote-failed-unit-codevipe-reset BAD=0 WARN=0 MODE=RESET_STALE_FAILED_STATE UNIT=codevipe-cloud-backup.service REASON=last_journal_success_and_no_timer ACTION=reset_failed_only\n76-1426:● homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image\n77:1499:2026-07-21T11:35:23Z\tBACKUP-DUAL-CLOUD-PREFLIGHT-059\tc6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479\t1\tFAIL\tbackup-audit\n78:1500:2026-07-21T12:28:26Z\tBACKUP-VM-DUMP-PIPELINE-073\tcd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b\t1\tFAIL\tbackup-audit\n79:1501:2026-07-21T12:42:53Z\tBACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075\ta9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936\t143\tFAIL\tbackup-audit\n80:1502:2026-07-21T12:56:52Z\tBACKUP-DRAIN-LOCAL-ARCHIVES-079\t531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32\t75\tFAIL\tbackup-remediation\n81:1503:2026-07-21T13:04:48Z\tBACKUP-FINAL-PROOF-082\t9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601\t127\tFAIL\tbackup-audit\n82:1505:2026-07-21T13:14:27Z\tCLUSTER-BACKUP-POSTCLEAN-PROOF-086\t601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da\t127\tFAIL\tbackup-cleanup\n83:1506:2026-07-21T13:15:21Z\tBACKUP-POLICY-CLOSEOUT-087\td79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52\t1\tFAIL\tbackup-cleanup\n84:1507:2026-07-21T13:17:20Z\tBACKUP-CLEANUP-FINAL-CLOSEOUT-088\t5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6\t1\tFAIL\tbackup-cleanup\n85-=== ACTIONABLE_P1 ===\n86-26:1:# HOMELAB ASSISTANT ERROR REGISTER\n87-28:77:21. Ошибка: nested PHP php -r дал Parse error на forum-prod.\n88-29:83:Команда 108 дала PHP Parse error на smtpHost и пустой proof-файл.\n--\n111-149:761:- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults.\n112-153:774:- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects.\n113:155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.\n114-171:839:- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`.\n115-180:865:- Symptom: error-register candidate construction stopped with RC1 before applying the candidate.\n116-183:876:- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis.\n117-201:948:BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative.\n118:203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.\n119-220:- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR.\n120-235:- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring.\n121-236:- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification.\n122-245:- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first.\n--\n129-459:STATUS=FAIL\n130-475:JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000\n131:486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800\n132-495:JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000\n133:508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000\n134-510:JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000\n135:513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800\n136-676:2026-07-08T09:26:36.7403618890 390 /var/lib/homelab-health/dockge-stale-stack-classification-after-agents.txt\n137-681:2026-07-08T09:26:36.7406026680 390 /var/lib/homelab-health/homepage-missing-card-proposals.txt\n138-684:2026-07-08T09:26:36.7408259230 390 /var/lib/homelab-health/homepage-missing-card-proposals-final.txt\n139-689:2026-07-08T09:26:36.7410552890 389 /var/lib/homelab-health/dockge-stale-archive-last.txt\n--\n142-1127:/var/lib/homelab-health/crypto-portfolio-remove-grt-recovery-audit.txt:1:STATUS=ERROR TS=2026-07-13T05:39:07Z TYPE=crypto-portfolio-remove-grt-recovery-audit BAD=1 COMMAND_PHYSICAL_LINES=1 ROOT_CAUSE=READINESS_ENDPOINT_MISMATCH RECOVERY_VERIFIED=NO TASK_COMPLETE=NO READY_FOR_GRT_APPLY_SCRIPT_FIX=NO AUDIT_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-recovery-audit-v2/20260713T053907Z\n143-1133:/var/lib/homelab-health/final-readiness.txt:1:STATUS=WARN TS=2026-07-21T05:31:30Z TYPE=final-readiness BAD=5 PASS=48 FAIL=5 REPORT=/var/lib/homelab-final-readiness/latest.md\n144:1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL\n145-1152:/var/lib/homelab-health/router-hard-power.txt:1:STATUS=ERROR TS=2026-07-16T06:20:34Z MODE=cycle REMOTE=[PRIVATE_IP] RC=74\n146-1162:/var/lib/homelab-health/private-vpn-hosts.txt.bak-ok-wrapper-20260624T132705Z:16:STATUS=ERROR\n147-1164:/var/lib/homelab-health/paperless-restore.txt:1:STATUS=STALE\n148-1165:/var/lib/homelab-health/paperless-restore.txt:20:FRESHNESS=STALE\n149:1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL\n150-1174:/var/lib/homelab-health/ungated-health-backlog.txt:1:STATUS=OK TS=2026-07-07T07:45:01Z TYPE=ungated-health-backlog BAD=0 WARN=1 MODE=POST_PVE03_ROOT_CLEANUP_TRIAGE TOTAL=8 SUPERSEDED=4 STALE=2 POLICY_GAP_NEXT_STAGE=1 ACTIONABLE_CAPACITY=1 PVE03_ROOT_USAGE_PERCENT=31 PVE03_STAGING_USAGE_PERCENT=83 REPORT=/var/lib/homelab-ungated-health-backlog/latest.md\n151-1190:/var/lib/homelab-health/mail-cloud-restore-drill.txt:1:STATUS=STALE\n152-1191:/var/lib/homelab-health/mail-cloud-restore-drill.txt:20:FRESHNESS=STALE\n153-1192:/var/lib/homelab-health/edge-restore-proofs-offhost.txt:1:STATUS=OK TS=20260624T190040Z TYPE=edge-restore-proofs-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge ITEMS=14 OK=14 ERROR=0 BYTES=226024287 SECRET_PRINTED=NO\n--\n156-1207:/var/lib/homelab-health/filebrowser-restore.txt:1:STATUS=STALE\n157-1208:/var/lib/homelab-health/filebrowser-restore.txt:20:FRESHNESS=STALE\n158:1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed\n159-1494:2026-07-21T08:03:07Z\tGITEA-PUSH-AUTH-001\tfcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94\t1\tFAIL\tgitea-runtime\n160-1495:2026-07-21T08:13:32Z\tGITEA-TOKEN-CLI-HELP-003\tf1d0f967eed6490ad822944a42108c82dc4b59c9fd2557d8c64624dcb6853cd4\t1\tFAIL\tgitea-runtime\n161-1496:2026-07-21T08:14:52Z\tGITEA-CLI-CAPABILITIES-004\t2cd6c4d28c8f646d8063105fca80b2f80d38c6a5db3a3eb76869de326b12d2d6\t1\tFAIL\tgitea-runtime\n162-1497:2026-07-21T09:49:51Z\tRUNTIME-RECEIVER-STATE-027\t0831002c78c26f35826cd7199995e335ceadcd39f08b9bac23882a0fbb59c8cc\t127\tFAIL\tgitea-runtime\n163-1498:2026-07-21T10:26:18Z\tRUNTIME-RECEIVER-CLEANUP-036\t65fbcfbce38a55723a92a8790429d6567389321f030f02cb4bf301cbe67e2531\t1\tFAIL\tgitea-runtime\n164:1499:2026-07-21T11:35:23Z\tBACKUP-DUAL-CLOUD-PREFLIGHT-059\tc6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479\t1\tFAIL\tbackup-audit\n165:1500:2026-07-21T12:28:26Z\tBACKUP-VM-DUMP-PIPELINE-073\tcd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b\t1\tFAIL\tbackup-audit\n166:1501:2026-07-21T12:42:53Z\tBACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075\ta9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936\t143\tFAIL\tbackup-audit\n167:1502:2026-07-21T12:56:52Z\tBACKUP-DRAIN-LOCAL-ARCHIVES-079\t531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32\t75\tFAIL\tbackup-remediation\n168:1503:2026-07-21T13:04:48Z\tBACKUP-FINAL-PROOF-082\t9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601\t127\tFAIL\tbackup-audit\n169-1504:2026-07-21T13:07:09Z\tCLUSTER-CLEANUP-INVENTORY-083\t09d3f14ca0205936a910dc7af7991474030273a47d76fa033ea6a3185c083922\t1\tFAIL\tcluster-cleanup\n170:1505:2026-07-21T13:14:27Z\tCLUSTER-BACKUP-POSTCLEAN-PROOF-086\t601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da\t127\tFAIL\tbackup-cleanup\n171:1506:2026-07-21T13:15:21Z\tBACKUP-POLICY-CLOSEOUT-087\td79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52\t1\tFAIL\tbackup-cleanup\n172:1507:2026-07-21T13:17:20Z\tBACKUP-CLEANUP-FINAL-CLOSEOUT-088\t5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6\t1\tFAIL\tbackup-cleanup\n173-=== ACTIONABLE_STORAGE ===\n174-40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.\n175-454:local dir active 102626232 41916788 55450180 40.84%\n176-=== OPEN_ERROR_REGISTER ===\n=== OPEN_ERROR_REGISTER ===\n=== OPEN_ERROR_REGISTER ===\nNONE\n=== RECENT_FAILED_COMMANDS_NORMALIZED ===\nCRITICAL_FIRST_ACTION=COMPLETE\n=== CLUSTER_QUORUM ===\nCluster information\n-------------------\nName: homelab\nConfig Version: 3\nTransport: knet\nSecure auth: on\n\nQuorum information\n------------------\nDate: Tue Jul 21 17:39:59 2026\nQuorum provider: corosync_votequorum\nNodes: 3\nNode ID: 0x00000001\nRing ID: 1.9d\nQuorate: Yes\n\nVotequorum information\n----------------------\nExpected votes: 3\nHighest expected: 3\nTotal votes: 3\nQuorum: 2 \nFlags: Quorate \n\nMembership information\n----------------------\n Nodeid Votes Name\n0x00000001 1 [PRIVATE_IP] (local)\n0x00000002 1 [PRIVATE_IP]\n0x00000003 1 [PRIVATE_IP]\n\nMembership information\n----------------------\n Nodeid Votes Name\n 1 1 pve01 (local)\n 2 1 pve03\n 3 1 pve02\n=== FAILED_UNITS ===\n UNIT LOAD ACTIVE SUB DESCRIPTION\n● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\n● homelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01\n● homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\n\nLegend: LOAD → Reflects whether the unit definition was properly loaded.\n ACTIVE → The high-level unit activation state, i.e. generalization of SUB.\n SUB → The low-level unit activation state, values depend on unit type.\n\n3 loaded units listed.\nTo show all installed unit files use 'systemctl list-unit-files'.\n--- UNIT=codevipe-cloud-backup.service ---\nResult=success\nNRestarts=0\nExecMainCode=0\nExecMainStatus=0\nId=codevipe-cloud-backup.service\nLoadState=not-found\nActiveState=inactive\nSubState=dead\nFragmentPath=\n-- No entries --\n--- UNIT=homelab-backup-matrix-refresh.service ---\nResult=exit-code\nNRestarts=0\nExecMainCode=1\nExecMainStatus=1\nId=homelab-backup-matrix-refresh.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nFragmentPath=/etc/systemd/system/homelab-backup-matrix-refresh.service\nJul 21 05:46:46 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nJul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_STATUS=FAIL\nJul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_TOTAL=41\nJul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_OK=30\nJul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_BAD=6\nJul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_DISABLED=5\nJul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 21 05:46:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 05:46:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 05:46:47 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nJul 21 06:06:46 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nJul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_STATUS=FAIL\nJul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_TOTAL=41\nJul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_OK=30\nJul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_BAD=6\nJul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_DISABLED=5\nJul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 21 06:06:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 06:06:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 06:06:47 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nJul 21 06:45:16 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nJul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_STATUS=FAIL\nJul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_TOTAL=41\nJul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_OK=30\nJul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_BAD=6\nJul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_DISABLED=5\nJul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 21 06:45:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 06:45:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 06:45:17 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nJul 21 07:05:43 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nJul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_STATUS=FAIL\nJul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_TOTAL=41\nJul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_OK=30\nJul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_BAD=6\nJul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_DISABLED=5\nJul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 21 07:05:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 07:05:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 07:05:44 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nJul 21 07:36:59 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nJul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_STATUS=FAIL\nJul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_TOTAL=41\nJul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_OK=30\nJul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_BAD=6\nJul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_DISABLED=5\nJul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 21 07:37:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 07:37:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 07:37:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nJul 21 08:07:26 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nJul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_STATUS=FAIL\nJul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_TOTAL=41\nJul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_OK=30\nJul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_BAD=6\nJul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_DISABLED=5\nJul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 21 08:07:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 08:07:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 08:07:27 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nJul 21 08:30:13 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nJul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_STATUS=FAIL\nJul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_TOTAL=41\nJul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_OK=30\nJul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_BAD=6\nJul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_DISABLED=5\nJul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 21 08:30:14 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 08:30:14 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 08:30:14 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nJul 21 08:41:14 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nJul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_STATUS=FAIL\nJul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_TOTAL=41\nJul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_OK=30\nJul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_BAD=6\nJul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_DISABLED=5\nJul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 21 08:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 08:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 08:41:15 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nJul 21 08:50:15 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nJul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_STATUS=FAIL\nJul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_TOTAL=41\nJul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_OK=30\nJul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_BAD=6\nJul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_DISABLED=5\nJul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 21 08:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 08:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 08:50:16 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nJul 21 10:02:41 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\nJul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_STATUS=FAIL\nJul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_TOTAL=41\nJul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_OK=30\nJul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_BAD=6\nJul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_UNMAPPED=0\nJul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_DISABLED=5\nJul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\nJul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\nJul 21 10:02:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 10:02:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 10:02:42 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n--- UNIT=homelab-cluster-admin-incident-engine-backup.service ---\nResult=success\nNRestarts=0\nExecMainCode=1\nExecMainStatus=0\nId=homelab-cluster-admin-incident-engine-backup.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nFragmentPath=/etc/systemd/system/homelab-cluster-admin-incident-engine-backup.service\nJul 13 07:33:14 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 13 07:33:24 pve01 homelab-cluster-admin-incident-engine-backup-run[2386922]: FLAT_CHUNKED_COPY_OK SIZE=1818333 PARTS=1\nJul 13 07:33:28 pve01 homelab-cluster-admin-incident-engine-backup-run[2387061]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T043314Z\nJul 13 07:33:37 pve01 homelab-cluster-admin-incident-engine-backup-run[2387287]: FLAT_CHUNKED_COPY_OK SIZE=1818333 PARTS=1\nJul 13 07:33:38 pve01 homelab-cluster-admin-incident-engine-backup-run[2387448]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T043314Z\nJul 13 07:33:39 pve01 homelab-cluster-admin-incident-engine-backup-run[2386743]: [SUCCESS] short flat application uploaded and verified\nJul 13 07:33:39 pve01 homelab-cluster-admin-incident-engine-backup-run[2386712]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 13 07:33:39 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 13 07:33:39 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 13 07:33:39 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.357s CPU time, 50.3M memory peak.\nJul 14 04:46:45 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 14 04:46:58 pve01 homelab-cluster-admin-incident-engine-backup-run[2275200]: FLAT_CHUNKED_COPY_OK SIZE=2493105 PARTS=1\nJul 14 04:46:58 pve01 homelab-cluster-admin-incident-engine-backup-run[2275366]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T014645Z\nJul 14 04:47:12 pve01 homelab-cluster-admin-incident-engine-backup-run[2275633]: FLAT_CHUNKED_COPY_OK SIZE=2493105 PARTS=1\nJul 14 04:47:12 pve01 homelab-cluster-admin-incident-engine-backup-run[2275815]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T014645Z\nJul 14 04:47:18 pve01 homelab-cluster-admin-incident-engine-backup-run[2275029]: [SUCCESS] short flat application uploaded and verified\nJul 14 04:47:18 pve01 homelab-cluster-admin-incident-engine-backup-run[2274993]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 14 04:47:18 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 14 04:47:18 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 14 04:47:18 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.491s CPU time, 54.3M memory peak.\nJul 14 04:54:18 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 14 04:54:33 pve01 homelab-cluster-admin-incident-engine-backup-run[2297844]: FLAT_CHUNKED_COPY_OK SIZE=2505726 PARTS=1\nJul 14 04:54:33 pve01 homelab-cluster-admin-incident-engine-backup-run[2298021]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T015418Z\nJul 14 04:54:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2298273]: FLAT_CHUNKED_COPY_OK SIZE=2505726 PARTS=1\nJul 14 04:54:50 pve01 homelab-cluster-admin-incident-engine-backup-run[2298435]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T015418Z\nJul 14 04:54:51 pve01 homelab-cluster-admin-incident-engine-backup-run[2297589]: [SUCCESS] short flat application uploaded and verified\nJul 14 04:54:51 pve01 homelab-cluster-admin-incident-engine-backup-run[2297401]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 14 04:54:51 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 14 04:54:51 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 14 04:54:51 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.453s CPU time, 55M memory peak.\nJul 14 07:32:36 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 14 07:32:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2862512]: FLAT_CHUNKED_COPY_OK SIZE=2584458 PARTS=1\nJul 14 07:32:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2862665]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T043236Z\nJul 14 07:33:02 pve01 homelab-cluster-admin-incident-engine-backup-run[2862911]: FLAT_CHUNKED_COPY_OK SIZE=2584458 PARTS=1\nJul 14 07:33:05 pve01 homelab-cluster-admin-incident-engine-backup-run[2863090]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T043236Z\nJul 14 07:33:06 pve01 homelab-cluster-admin-incident-engine-backup-run[2862304]: [SUCCESS] short flat application uploaded and verified\nJul 14 07:33:06 pve01 homelab-cluster-admin-incident-engine-backup-run[2862165]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 14 07:33:06 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 14 07:33:06 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 14 07:33:06 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.436s CPU time, 54.9M memory peak.\nJul 14 09:43:27 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 14 09:43:39 pve01 homelab-cluster-admin-incident-engine-backup-run[3315449]: FLAT_CHUNKED_COPY_OK SIZE=2651545 PARTS=1\nJul 14 09:43:42 pve01 homelab-cluster-admin-incident-engine-backup-run[3315701]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T064327Z\nJul 14 09:43:51 pve01 homelab-cluster-admin-incident-engine-backup-run[3315929]: FLAT_CHUNKED_COPY_OK SIZE=2651545 PARTS=1\nJul 14 09:43:52 pve01 homelab-cluster-admin-incident-engine-backup-run[3316098]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T064327Z\nJul 14 09:43:55 pve01 homelab-cluster-admin-incident-engine-backup-run[3315291]: [SUCCESS] short flat application uploaded and verified\nJul 14 09:43:55 pve01 homelab-cluster-admin-incident-engine-backup-run[3315256]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 14 09:43:55 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 14 09:43:55 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 14 09:43:55 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.483s CPU time, 55.4M memory peak.\nJul 15 07:35:00 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 15 07:35:11 pve01 homelab-cluster-admin-incident-engine-backup-run[3512923]: FLAT_CHUNKED_COPY_OK SIZE=3303993 PARTS=1\nJul 15 07:35:11 pve01 homelab-cluster-admin-incident-engine-backup-run[3513093]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260715T043500Z\nJul 15 07:35:30 pve01 homelab-cluster-admin-incident-engine-backup-run[3513650]: FLAT_CHUNKED_COPY_OK SIZE=3303993 PARTS=1\nJul 15 07:35:31 pve01 homelab-cluster-admin-incident-engine-backup-run[3513799]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260715T043500Z\nJul 15 07:35:34 pve01 homelab-cluster-admin-incident-engine-backup-run[3512747]: [SUCCESS] short flat application uploaded and verified\nJul 15 07:35:34 pve01 homelab-cluster-admin-incident-engine-backup-run[3512710]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 15 07:35:34 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 15 07:35:34 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 15 07:35:34 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.396s CPU time, 59.3M memory peak.\nJul 16 07:32:14 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 16 07:32:23 pve01 homelab-cluster-admin-incident-engine-backup-run[95619]: FLAT_CHUNKED_COPY_OK SIZE=4089435 PARTS=1\nJul 16 07:32:24 pve01 homelab-cluster-admin-incident-engine-backup-run[95775]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260716T043214Z\nJul 16 07:32:34 pve01 homelab-cluster-admin-incident-engine-backup-run[95997]: FLAT_CHUNKED_COPY_OK SIZE=4089435 PARTS=1\nJul 16 07:32:35 pve01 homelab-cluster-admin-incident-engine-backup-run[96153]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260716T043214Z\nJul 16 07:32:36 pve01 homelab-cluster-admin-incident-engine-backup-run[95482]: [SUCCESS] short flat application uploaded and verified\nJul 16 07:32:36 pve01 homelab-cluster-admin-incident-engine-backup-run[95370]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 16 07:32:36 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 16 07:32:36 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 16 07:32:36 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.646s CPU time, 66.1M memory peak.\n-- Boot 9fe27497bd1642a386cb88c33185f42f --\nJul 17 07:30:49 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 17 07:31:02 pve01 homelab-cluster-admin-incident-engine-backup-run[4051907]: FLAT_CHUNKED_COPY_OK SIZE=4782437 PARTS=1\nJul 17 07:31:03 pve01 homelab-cluster-admin-incident-engine-backup-run[4052105]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260717T043049Z\nJul 17 07:31:17 pve01 homelab-cluster-admin-incident-engine-backup-run[4052291]: FLAT_CHUNKED_COPY_OK SIZE=4782437 PARTS=1\nJul 17 07:31:20 pve01 homelab-cluster-admin-incident-engine-backup-run[4052752]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260717T043049Z\nJul 17 07:31:24 pve01 homelab-cluster-admin-incident-engine-backup-run[4051765]: [SUCCESS] short flat application uploaded and verified\nJul 17 07:31:24 pve01 homelab-cluster-admin-incident-engine-backup-run[4051685]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 17 07:31:24 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 17 07:31:24 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 17 07:31:24 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.642s CPU time, 68.6M memory peak.\nJul 18 07:31:41 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 18 07:31:54 pve01 homelab-cluster-admin-incident-engine-backup-run[685264]: FLAT_CHUNKED_COPY_OK SIZE=5630582 PARTS=1\nJul 18 07:31:55 pve01 homelab-cluster-admin-incident-engine-backup-run[685475]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260718T043141Z\nJul 18 07:32:06 pve01 homelab-cluster-admin-incident-engine-backup-run[685660]: FLAT_CHUNKED_COPY_OK SIZE=5630582 PARTS=1\nJul 18 07:32:07 pve01 homelab-cluster-admin-incident-engine-backup-run[685846]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260718T043141Z\nJul 18 07:32:08 pve01 homelab-cluster-admin-incident-engine-backup-run[685098]: [SUCCESS] short flat application uploaded and verified\nJul 18 07:32:08 pve01 homelab-cluster-admin-incident-engine-backup-run[684962]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 18 07:32:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 18 07:32:08 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 18 07:32:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.695s CPU time, 73.5M memory peak.\nJul 19 07:26:05 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 19 07:26:14 pve01 homelab-cluster-admin-incident-engine-backup-run[1415764]: FLAT_CHUNKED_COPY_OK SIZE=6386016 PARTS=1\nJul 19 07:26:14 pve01 homelab-cluster-admin-incident-engine-backup-run[1415903]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260719T042605Z\nJul 19 07:26:31 pve01 homelab-cluster-admin-incident-engine-backup-run[1416145]: FLAT_CHUNKED_COPY_OK SIZE=6386016 PARTS=1\nJul 19 07:26:31 pve01 homelab-cluster-admin-incident-engine-backup-run[1416534]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260719T042605Z\nJul 19 07:26:32 pve01 homelab-cluster-admin-incident-engine-backup-run[1415638]: [SUCCESS] short flat application uploaded and verified\nJul 19 07:26:32 pve01 homelab-cluster-admin-incident-engine-backup-run[1415567]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 19 07:26:32 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 19 07:26:32 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 19 07:26:32 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.704s CPU time, 78.3M memory peak.\nJul 20 07:37:35 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 20 07:37:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2212986]: FLAT_CHUNKED_COPY_OK SIZE=7183272 PARTS=1\nJul 20 07:37:52 pve01 homelab-cluster-admin-incident-engine-backup-run[2213181]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260720T043735Z\nJul 20 07:38:04 pve01 homelab-cluster-admin-incident-engine-backup-run[2213436]: FLAT_CHUNKED_COPY_OK SIZE=7183272 PARTS=1\nJul 20 07:38:07 pve01 homelab-cluster-admin-incident-engine-backup-run[2213580]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260720T043735Z\nJul 20 07:38:08 pve01 homelab-cluster-admin-incident-engine-backup-run[2212839]: [SUCCESS] short flat application uploaded and verified\nJul 20 07:38:08 pve01 homelab-cluster-admin-incident-engine-backup-run[2212719]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 20 07:38:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 20 07:38:08 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 20 07:38:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.798s CPU time, 83M memory peak.\nJul 21 07:36:14 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...\nJul 21 07:36:26 pve01 homelab-cluster-admin-incident-engine-backup-run[3010900]: FLAT_CHUNKED_COPY_OK SIZE=8130454 PARTS=1\nJul 21 07:36:27 pve01 homelab-cluster-admin-incident-engine-backup-run[3011053]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260721T043614Z\nJul 21 07:36:36 pve01 homelab-cluster-admin-incident-engine-backup-run[3011255]: FLAT_CHUNKED_COPY_OK SIZE=8130454 PARTS=1\nJul 21 07:36:37 pve01 homelab-cluster-admin-incident-engine-backup-run[3011413]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260721T043614Z\nJul 21 07:36:40 pve01 homelab-cluster-admin-incident-engine-backup-run[3010739]: [SUCCESS] short flat application uploaded and verified\nJul 21 07:36:40 pve01 homelab-cluster-admin-incident-engine-backup-run[3010566]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK\nJul 21 07:36:40 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.\nJul 21 07:36:40 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.\nJul 21 07:36:40 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.760s CPU time, 88.3M memory peak.\n--- UNIT=homelab-cluster-admin-pull.service ---\nResult=exit-code\nNRestarts=0\nExecMainCode=1\nExecMainStatus=1\nId=homelab-cluster-admin-pull.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nFragmentPath=/etc/systemd/system/homelab-cluster-admin-pull.service\nJul 21 01:06:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 01:11:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 01:11:36 pve01 homelab-cluster-admin-pull[1709417]: STATUS=WARN TS=2026-07-20T22:11:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 01:11:36 pve01 homelab-cluster-admin-pull[1709418]: STATUS=WARN TS=2026-07-20T22:11:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 01:11:36 pve01 homelab-cluster-admin-pull[1709419]: STATUS=WARN TS=2026-07-20T22:11:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 01:11:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:11:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 01:11:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 01:16:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 01:16:36 pve01 homelab-cluster-admin-pull[1733408]: STATUS=WARN TS=2026-07-20T22:16:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 01:16:36 pve01 homelab-cluster-admin-pull[1733409]: STATUS=WARN TS=2026-07-20T22:16:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 01:16:36 pve01 homelab-cluster-admin-pull[1733410]: STATUS=WARN TS=2026-07-20T22:16:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 01:16:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:16:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 01:16:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 01:21:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 01:21:36 pve01 homelab-cluster-admin-pull[1749007]: STATUS=WARN TS=2026-07-20T22:21:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 01:21:36 pve01 homelab-cluster-admin-pull[1749008]: STATUS=WARN TS=2026-07-20T22:21:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 01:21:36 pve01 homelab-cluster-admin-pull[1749009]: STATUS=WARN TS=2026-07-20T22:21:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 01:21:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:21:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 01:21:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 01:26:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 01:26:36 pve01 homelab-cluster-admin-pull[1764598]: STATUS=WARN TS=2026-07-20T22:26:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 01:26:36 pve01 homelab-cluster-admin-pull[1764599]: STATUS=WARN TS=2026-07-20T22:26:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 01:26:36 pve01 homelab-cluster-admin-pull[1764600]: STATUS=WARN TS=2026-07-20T22:26:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 01:26:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:26:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 01:26:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 01:31:45 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 01:31:52 pve01 homelab-cluster-admin-pull[1788839]: STATUS=WARN TS=2026-07-20T22:31:45Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 01:31:52 pve01 homelab-cluster-admin-pull[1788840]: STATUS=WARN TS=2026-07-20T22:31:45Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 01:31:52 pve01 homelab-cluster-admin-pull[1788841]: STATUS=WARN TS=2026-07-20T22:31:45Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 01:31:52 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:31:52 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 01:31:52 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 01:37:07 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 01:37:07 pve01 homelab-cluster-admin-pull[1804555]: STATUS=WARN TS=2026-07-20T22:37:07Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 01:37:07 pve01 homelab-cluster-admin-pull[1804556]: STATUS=WARN TS=2026-07-20T22:37:07Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 01:37:07 pve01 homelab-cluster-admin-pull[1804557]: STATUS=WARN TS=2026-07-20T22:37:07Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 01:37:07 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:37:07 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 01:37:07 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 01:42:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 01:42:15 pve01 homelab-cluster-admin-pull[1820229]: STATUS=WARN TS=2026-07-20T22:42:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 01:42:15 pve01 homelab-cluster-admin-pull[1820230]: STATUS=WARN TS=2026-07-20T22:42:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 01:42:15 pve01 homelab-cluster-admin-pull[1820231]: STATUS=WARN TS=2026-07-20T22:42:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 01:42:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:42:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 01:42:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 01:47:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 01:47:15 pve01 homelab-cluster-admin-pull[1844123]: STATUS=WARN TS=2026-07-20T22:47:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 01:47:15 pve01 homelab-cluster-admin-pull[1844124]: STATUS=WARN TS=2026-07-20T22:47:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 01:47:15 pve01 homelab-cluster-admin-pull[1844125]: STATUS=WARN TS=2026-07-20T22:47:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 01:47:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:47:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 01:47:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 01:52:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 01:52:15 pve01 homelab-cluster-admin-pull[1859657]: STATUS=WARN TS=2026-07-20T22:52:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 01:52:15 pve01 homelab-cluster-admin-pull[1859658]: STATUS=WARN TS=2026-07-20T22:52:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 01:52:15 pve01 homelab-cluster-admin-pull[1859659]: STATUS=WARN TS=2026-07-20T22:52:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 01:52:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:52:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 01:52:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 01:57:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 01:57:15 pve01 homelab-cluster-admin-pull[1875102]: STATUS=WARN TS=2026-07-20T22:57:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 01:57:15 pve01 homelab-cluster-admin-pull[1875103]: STATUS=WARN TS=2026-07-20T22:57:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 01:57:15 pve01 homelab-cluster-admin-pull[1875104]: STATUS=WARN TS=2026-07-20T22:57:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 01:57:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:57:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 01:57:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 02:02:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 02:02:15 pve01 homelab-cluster-admin-pull[1899107]: STATUS=WARN TS=2026-07-20T23:02:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 02:02:15 pve01 homelab-cluster-admin-pull[1899108]: STATUS=WARN TS=2026-07-20T23:02:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 02:02:15 pve01 homelab-cluster-admin-pull[1899109]: STATUS=WARN TS=2026-07-20T23:02:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 02:02:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:02:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 02:02:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 02:07:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 02:07:15 pve01 homelab-cluster-admin-pull[1914657]: STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 02:07:15 pve01 homelab-cluster-admin-pull[1914658]: STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 02:07:15 pve01 homelab-cluster-admin-pull[1914659]: STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 02:07:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:07:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 02:07:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 02:12:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 02:12:15 pve01 homelab-cluster-admin-pull[1930715]: STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 02:12:15 pve01 homelab-cluster-admin-pull[1930716]: STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 02:12:15 pve01 homelab-cluster-admin-pull[1930717]: STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 02:12:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:12:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 02:12:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 02:14:04 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 02:14:11 pve01 homelab-cluster-admin-pull[1932800]: STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 02:14:11 pve01 homelab-cluster-admin-pull[1932801]: STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 02:14:11 pve01 homelab-cluster-admin-pull[1932802]: STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 02:14:11 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:14:11 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 02:14:11 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 02:20:13 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 02:20:20 pve01 homelab-cluster-admin-pull[1958372]: STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 02:20:20 pve01 homelab-cluster-admin-pull[1958373]: STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 02:20:20 pve01 homelab-cluster-admin-pull[1958374]: STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 02:20:20 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:20:20 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 02:20:20 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 02:35:05 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 02:35:12 pve01 homelab-cluster-admin-pull[2013630]: STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 02:35:12 pve01 homelab-cluster-admin-pull[2013631]: STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 02:35:12 pve01 homelab-cluster-admin-pull[2013632]: STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 02:35:12 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:35:12 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 02:35:12 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\nJul 21 02:36:57 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015088]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015089]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md\nJul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015090]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer\nJul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.\nJul 21 02:37:04 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.\n--- UNIT=homelab-edge-vm-offhost-freshness.timer ---\nResult=success\nId=homelab-edge-vm-offhost-freshness.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nFragmentPath=/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer\nJul 16 09:20:43 pve01 systemd[1]: homelab-edge-vm-offhost-freshness.timer: Deactivated successfully.\nJul 16 09:20:43 pve01 systemd[1]: Stopped homelab-edge-vm-offhost-freshness.timer - Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump.\n-- Boot 9fe27497bd1642a386cb88c33185f42f --\nJul 16 11:38:48 pve01 systemd[1]: Started homelab-edge-vm-offhost-freshness.timer - Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump.\n--- UNIT=homelab-immutable-offline-verify.service ---\nResult=success\nNRestarts=0\nExecMainCode=0\nExecMainStatus=0\nId=homelab-immutable-offline-verify.service\nLoadState=not-found\nActiveState=inactive\nSubState=dead\nFragmentPath=\n-- No entries --\n--- UNIT=homelab-vm160-remote-backup.service ---\nResult=exit-code\nNRestarts=0\nExecMainCode=1\nExecMainStatus=1\nId=homelab-vm160-remote-backup.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nFragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service\nJul 18 01:02:41 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 18 01:15:43 pve01 homelab-vm160-remote-backup[3491061]: VM160_TARGET_FINALIZE_STATUS=OK\nJul 18 01:19:10 pve01 homelab-vm160-remote-backup[3530071]: VM160_TARGET_PROMOTE_STATUS=OK\nJul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: VM160_REMOTE_BACKUP_STATUS=OK\nJul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z\nJul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z/vzdump-qemu-160-20260717T220242Z.vma.zst\nJul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: SIZE_BYTES=84993482488\nJul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: SHA256=a9562d8b60d7fd5e7ec73bc43d252cfea80ed12c05b32cca0a73d7bb88532411\nJul 18 01:19:10 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully.\nJul 18 01:19:10 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nJul 18 01:19:10 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak.\nJul 18 01:38:18 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 18 01:51:18 pve01 homelab-vm160-remote-backup[3609200]: VM160_TARGET_FINALIZE_STATUS=OK\nJul 18 01:54:48 pve01 homelab-vm160-remote-backup[3648126]: VM160_TARGET_PROMOTE_STATUS=OK\nJul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: VM160_REMOTE_BACKUP_STATUS=OK\nJul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z\nJul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst\nJul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: SIZE_BYTES=84995216465\nJul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72\nJul 18 01:54:48 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully.\nJul 18 01:54:48 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nJul 18 01:54:48 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak.\nJul 18 04:16:25 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 18 04:16:27 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 18 04:16:27 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 18 04:16:27 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nJul 19 04:19:14 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 19 04:19:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 04:19:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 19 04:19:16 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nJul 20 04:14:14 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 20 04:14:16 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nJul 20 19:28:57 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 20 19:28:59 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nJul 21 02:14:11 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 21 02:14:13 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:14:13 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 21 02:14:13 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nJul 21 02:20:20 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 21 02:20:21 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:20:21 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 21 02:20:21 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\nJul 21 02:39:12 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...\nJul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 21 02:39:14 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.\n=== PROXMOX_STORAGE ===\nName Type Status Total (KiB) Used (KiB) Available (KiB) %\nlocal dir active 102626232 41936392 55430576 40.86%\nlocal-lvm lvmthin active 848797696 280188119 568609576 33.01%\nFilesystem Type Size Used Avail Use% Mounted on\nudev devtmpfs 15G 0 15G 0% /dev\ntmpfs tmpfs 3.2G 3.8M 3.2G 1% /run\n/dev/mapper/pve-root ext4 98G 40G 53G 44% /\ntmpfs tmpfs 16G 66M 16G 1% /dev/shm\nefivarfs efivarfs 192K 135K 53K 72% /sys/firmware/efi/efivars\ntmpfs tmpfs 5.0M 0 5.0M 0% /run/lock\ntmpfs tmpfs 1.0M 0 1.0M 0% /run/credentials/systemd-journald.service\ntmpfs tmpfs 16G 48M 16G 1% /tmp\n/dev/nvme0n1p2 vfat 1022M 9.1M 1013M 1% /boot/efi\n/dev/mapper/pve-homelab--staging ext4 295G 148G 133G 53% /mnt/staging\n/dev/fuse fuse 128M 1.4M 127M 2% /etc/pve\nshm tmpfs 63M 0 63M 0% /var/lib/containers/storage/overlay-containers/fbb240ea5b547fa140d8edbc3b16daaf4b28a21c7ee424abedab068865a95d5b/userdata/shm\noverlay overlay 98G 40G 53G 44% /var/lib/containers/storage/overlay/4f68ccfa2802b35db2a4c6b396e477f756cb85aa42fa818dd111745ec503616b/merged\ntmpfs tmpfs 1.0M 0 1.0M 0% /run/credentials/getty@tty1.service\ntmpfs tmpfs 3.2G 4.0K 3.2G 1% /run/user/0\nFilesystem Inodes IUsed IFree IUse% Mounted on\nudev 3.6M 733 3.6M 1% /dev\ntmpfs 3.9M 1.4K 3.9M 1% /run\n/dev/mapper/pve-root 6.3M 128K 6.2M 2% /\ntmpfs 3.9M 137 3.9M 1% /dev/shm\nefivarfs 0 0 0 - /sys/firmware/efi/efivars\ntmpfs 3.9M 32 3.9M 1% /run/lock\ntmpfs 1.0K 1 1023 1% /run/credentials/systemd-journald.service\ntmpfs 1.0M 3.5K 1021K 1% /tmp\n/dev/nvme0n1p2 0 0 0 - /boot/efi\n/dev/mapper/pve-homelab--staging 19M 4.3K 19M 1% /mnt/staging\n/dev/fuse 256K 89 256K 1% /etc/pve\nshm 3.9M 1 3.9M 1% /var/lib/containers/storage/overlay-containers/fbb240ea5b547fa140d8edbc3b16daaf4b28a21c7ee424abedab068865a95d5b/userdata/shm\noverlay 6.3M 128K 6.2M 2% /var/lib/containers/storage/overlay/4f68ccfa2802b35db2a4c6b396e477f756cb85aa42fa818dd111745ec503616b/merged\ntmpfs 1.0K 1 1023 1% /run/credentials/getty@tty1.service\ntmpfs 796K 28 796K 1% /run/user/0\n=== STORAGE_BACKENDS ===\nno pools available\nPersonalities : \nunused devices: \nFile descriptor 9 (/var/lib/homelab-chat-run/state/runner.lock) leaked on pvs invocation. Parent PID 918816: /usr/bin/bash\n /dev/nvme0n1p3 <952.00g 16.00g a-- \nFile descriptor 9 (/var/lib/homelab-chat-run/state/runner.lock) leaked on vgs invocation. Parent PID 918816: /usr/bin/bash\n pve <952.00g 16.00g wz--n-\nFile descriptor 9 (/var/lib/homelab-chat-run/state/runner.lock) leaked on lvs invocation. Parent PID 918816: /usr/bin/bash\n data pve <809.48g twi-aotz-- 33.01 1.42 \n [data_tdata] pve <809.48g Twi-ao---- \n [data_tmeta] pve <8.26g ewi-ao---- \n homelab-staging pve 300.00g Vwi-aotz-- 66.86 \n [lvol0_pmspare] pve <8.26g ewi------- \n root pve 100.00g -wi-ao---- \n snap_vm-150-disk-0_before-nc-freshcheck-20260614-233339 pve 64.00g Vri---tz-k \n snap_vm-150-disk-0_nc-posttune-20260614-2343 pve 64.00g Vri---tz-k \n swap pve 10.00g -wi-ao---- \n vm-110-disk-0 pve 8.00g Vwi-aotz-- 26.63 \n vm-112-disk-0 pve 8.00g Vwi-aotz-- 13.54 \n vm-150-cloudinit pve 4.00m Vwi-aotz-- 9.38 \n vm-150-disk-0 pve 64.00g Vwi-aotz-- 22.72 \n vm-170-cloudinit pve 4.00m Vwi-aotz-- 9.38 \n vm-170-disk-0 pve 120.00g Vwi-aotz-- 32.00 \n vm-171-cloudinit pve 4.00m Vwi-aotz-- 9.38 \n vm-171-disk-0 pve 120.00g Vwi-aotz-- 7.05 \n=== KERNEL_CRITICAL_LAST_24H ===\nJul 20 19:31:56 pve01 systemd-sysv-generator[535545]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 20 19:31:56 pve01 systemd-sysv-generator[535545]: Please update package to include a native systemd unit file.\nJul 20 19:31:56 pve01 systemd-sysv-generator[535545]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 20 19:31:56 pve01 systemd-sysv-generator[535601]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 20 19:31:56 pve01 systemd-sysv-generator[535601]: Please update package to include a native systemd unit file.\nJul 20 19:31:56 pve01 systemd-sysv-generator[535601]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 20 19:31:57 pve01 systemd-sysv-generator[535675]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 20 19:31:57 pve01 systemd-sysv-generator[535675]: Please update package to include a native systemd unit file.\nJul 20 19:31:57 pve01 systemd-sysv-generator[535675]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 20 19:31:57 pve01 systemd-sysv-generator[535733]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 20 19:31:57 pve01 systemd-sysv-generator[535733]: Please update package to include a native systemd unit file.\nJul 20 19:31:57 pve01 systemd-sysv-generator[535733]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 20 22:15:38 pve01 systemd-sysv-generator[1097452]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 20 22:15:38 pve01 systemd-sysv-generator[1097452]: Please update package to include a native systemd unit file.\nJul 20 22:15:38 pve01 systemd-sysv-generator[1097452]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 20 22:15:39 pve01 systemd-sysv-generator[1097526]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 20 22:15:39 pve01 systemd-sysv-generator[1097526]: Please update package to include a native systemd unit file.\nJul 20 22:15:39 pve01 systemd-sysv-generator[1097526]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 20 23:59:14 pve01 systemd-sysv-generator[1457483]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 20 23:59:14 pve01 systemd-sysv-generator[1457483]: Please update package to include a native systemd unit file.\nJul 20 23:59:14 pve01 systemd-sysv-generator[1457483]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 20 23:59:14 pve01 systemd-sysv-generator[1457536]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 20 23:59:14 pve01 systemd-sysv-generator[1457536]: Please update package to include a native systemd unit file.\nJul 20 23:59:14 pve01 systemd-sysv-generator[1457536]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 00:09:32 pve01 systemd-sysv-generator[1497895]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 00:09:32 pve01 systemd-sysv-generator[1497895]: Please update package to include a native systemd unit file.\nJul 21 00:09:32 pve01 systemd-sysv-generator[1497895]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 00:09:32 pve01 systemd-sysv-generator[1497977]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 00:09:32 pve01 systemd-sysv-generator[1497977]: Please update package to include a native systemd unit file.\nJul 21 00:09:32 pve01 systemd-sysv-generator[1497977]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 00:33:12 pve01 systemd-sysv-generator[1580930]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 00:33:12 pve01 systemd-sysv-generator[1580930]: Please update package to include a native systemd unit file.\nJul 21 00:33:12 pve01 systemd-sysv-generator[1580930]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 00:33:12 pve01 systemd-sysv-generator[1580983]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 00:33:12 pve01 systemd-sysv-generator[1580983]: Please update package to include a native systemd unit file.\nJul 21 00:33:12 pve01 systemd-sysv-generator[1580983]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917427]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917427]: Please update package to include a native systemd unit file.\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917427]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917497]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917497]: Please update package to include a native systemd unit file.\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917497]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917569]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917569]: Please update package to include a native systemd unit file.\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917569]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917640]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917640]: Please update package to include a native systemd unit file.\nJul 21 02:10:30 pve01 systemd-sysv-generator[1917640]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:11:53 pve01 systemd-sysv-generator[1930242]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:11:53 pve01 systemd-sysv-generator[1930242]: Please update package to include a native systemd unit file.\nJul 21 02:11:53 pve01 systemd-sysv-generator[1930242]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:11:54 pve01 systemd-sysv-generator[1930305]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:11:54 pve01 systemd-sysv-generator[1930305]: Please update package to include a native systemd unit file.\nJul 21 02:11:54 pve01 systemd-sysv-generator[1930305]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:11:54 pve01 systemd-sysv-generator[1930369]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:11:54 pve01 systemd-sysv-generator[1930369]: Please update package to include a native systemd unit file.\nJul 21 02:11:54 pve01 systemd-sysv-generator[1930369]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:14:02 pve01 systemd-sysv-generator[1932245]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:14:02 pve01 systemd-sysv-generator[1932245]: Please update package to include a native systemd unit file.\nJul 21 02:14:02 pve01 systemd-sysv-generator[1932245]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:14:03 pve01 systemd-sysv-generator[1932308]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:14:03 pve01 systemd-sysv-generator[1932308]: Please update package to include a native systemd unit file.\nJul 21 02:14:03 pve01 systemd-sysv-generator[1932308]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:14:03 pve01 systemd-sysv-generator[1932371]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:14:03 pve01 systemd-sysv-generator[1932371]: Please update package to include a native systemd unit file.\nJul 21 02:14:03 pve01 systemd-sysv-generator[1932371]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:14:04 pve01 systemd-sysv-generator[1932629]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:14:04 pve01 systemd-sysv-generator[1932629]: Please update package to include a native systemd unit file.\nJul 21 02:14:04 pve01 systemd-sysv-generator[1932629]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:14:11 pve01 systemd-sysv-generator[1932867]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:14:11 pve01 systemd-sysv-generator[1932867]: Please update package to include a native systemd unit file.\nJul 21 02:14:11 pve01 systemd-sysv-generator[1932867]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:14:13 pve01 systemd-sysv-generator[1932964]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:14:13 pve01 systemd-sysv-generator[1932964]: Please update package to include a native systemd unit file.\nJul 21 02:14:13 pve01 systemd-sysv-generator[1932964]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:16:57 pve01 systemd-sysv-generator[1955003]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:16:57 pve01 systemd-sysv-generator[1955003]: Please update package to include a native systemd unit file.\nJul 21 02:16:57 pve01 systemd-sysv-generator[1955003]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:16:57 pve01 systemd-sysv-generator[1955071]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:16:57 pve01 systemd-sysv-generator[1955071]: Please update package to include a native systemd unit file.\nJul 21 02:16:57 pve01 systemd-sysv-generator[1955071]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:16:57 pve01 systemd-sysv-generator[1955141]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:16:57 pve01 systemd-sysv-generator[1955141]: Please update package to include a native systemd unit file.\nJul 21 02:16:57 pve01 systemd-sysv-generator[1955141]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:20:11 pve01 systemd-sysv-generator[1957829]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:20:11 pve01 systemd-sysv-generator[1957829]: Please update package to include a native systemd unit file.\nJul 21 02:20:11 pve01 systemd-sysv-generator[1957829]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:20:11 pve01 systemd-sysv-generator[1957891]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:20:11 pve01 systemd-sysv-generator[1957891]: Please update package to include a native systemd unit file.\nJul 21 02:20:11 pve01 systemd-sysv-generator[1957891]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:20:12 pve01 systemd-sysv-generator[1957959]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:20:12 pve01 systemd-sysv-generator[1957959]: Please update package to include a native systemd unit file.\nJul 21 02:20:12 pve01 systemd-sysv-generator[1957959]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:20:13 pve01 systemd-sysv-generator[1958205]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:20:13 pve01 systemd-sysv-generator[1958205]: Please update package to include a native systemd unit file.\nJul 21 02:20:13 pve01 systemd-sysv-generator[1958205]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:20:20 pve01 systemd-sysv-generator[1958431]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:20:20 pve01 systemd-sysv-generator[1958431]: Please update package to include a native systemd unit file.\nJul 21 02:20:20 pve01 systemd-sysv-generator[1958431]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:20:21 pve01 systemd-sysv-generator[1958526]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:20:21 pve01 systemd-sysv-generator[1958526]: Please update package to include a native systemd unit file.\nJul 21 02:20:21 pve01 systemd-sysv-generator[1958526]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:22:02 pve01 systemd-sysv-generator[1971264]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:22:02 pve01 systemd-sysv-generator[1971264]: Please update package to include a native systemd unit file.\nJul 21 02:22:02 pve01 systemd-sysv-generator[1971264]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:22:02 pve01 systemd-sysv-generator[1971316]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:22:02 pve01 systemd-sysv-generator[1971316]: Please update package to include a native systemd unit file.\nJul 21 02:22:02 pve01 systemd-sysv-generator[1971316]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:22:02 pve01 systemd-sysv-generator[1971377]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:22:02 pve01 systemd-sysv-generator[1971377]: Please update package to include a native systemd unit file.\nJul 21 02:22:02 pve01 systemd-sysv-generator[1971377]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:23:50 pve01 systemd-sysv-generator[1973034]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:23:50 pve01 systemd-sysv-generator[1973034]: Please update package to include a native systemd unit file.\nJul 21 02:23:50 pve01 systemd-sysv-generator[1973034]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:23:50 pve01 systemd-sysv-generator[1973101]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:23:50 pve01 systemd-sysv-generator[1973101]: Please update package to include a native systemd unit file.\nJul 21 02:23:50 pve01 systemd-sysv-generator[1973101]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 02:23:50 pve01 systemd-sysv-generator[1973172]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.\nJul 21 02:23:50 pve01 systemd-sysv-generator[1973172]: Please update package to include a native systemd unit file.\nJul 21 02:23:50 pve01 systemd-sysv-generator[1973172]: ! This compatibility logic is deprecated, expect removal soon. !\nJul 21 03:30:02 pve01 kernel: EXT4-fs (dm-15): write access unavailable, skipping orphan cleanup\nJul 21 03:30:10 pve01 kernel: EXT4-fs (dm-15): write access unavailable, skipping orphan cleanup\n=== BACKUP_HEALTH ===\n dev-disk-by\\x2did-dm\\x2dname\\x2dpve\\x2dvm\\x2d\\x2d150\\x2d\\x2dcloudinit.device loaded active plugged /dev/disk/by-id/dm-name-pve-vm--150--cloudinit\n dev-disk-by\\x2did-dm\\x2dname\\x2dpve\\x2dvm\\x2d\\x2d170\\x2d\\x2dcloudinit.device loaded active plugged /dev/disk/by-id/dm-name-pve-vm--170--cloudinit\n dev-disk-by\\x2did-dm\\x2dname\\x2dpve\\x2dvm\\x2d\\x2d171\\x2d\\x2dcloudinit.device loaded active plugged /dev/disk/by-id/dm-name-pve-vm--171--cloudinit\n dev-mapper-pve\\x2dvm\\x2d\\x2d150\\x2d\\x2dcloudinit.device loaded active plugged /dev/mapper/pve-vm--150--cloudinit\n dev-mapper-pve\\x2dvm\\x2d\\x2d170\\x2d\\x2dcloudinit.device loaded active plugged /dev/mapper/pve-vm--170--cloudinit\n dev-mapper-pve\\x2dvm\\x2d\\x2d171\\x2d\\x2dcloudinit.device loaded active plugged /dev/mapper/pve-vm--171--cloudinit\n dev-pve-vm\\x2d150\\x2dcloudinit.device loaded active plugged /dev/pve/vm-150-cloudinit\n dev-pve-vm\\x2d170\\x2dcloudinit.device loaded active plugged /dev/pve/vm-170-cloudinit\n dev-pve-vm\\x2d171\\x2dcloudinit.device loaded active plugged /dev/pve/vm-171-cloudinit\n dpkg-db-backup.service loaded inactive dead Daily dpkg database backup service\n filebrowser-backup.service loaded inactive dead File Browser backup\n homelab-alertmanager-backup.service loaded inactive dead Alertmanager encrypted two-cloud application backup\n homelab-appbackup-dockge-stacks.service loaded inactive dead Homelab app-level backup for dockge-stacks\n homelab-appbackup-mariadb-batch.service loaded inactive dead Homelab MariaDB app batch backup\n homelab-appbackup-npmplus-edge.service loaded inactive dead Homelab app-level backup for npmplus-edge\n homelab-appbackup-postgres-batch.service loaded inactive dead Homelab Postgres app batch backup\n homelab-appbackup-sqlite-batch.service loaded inactive dead Homelab SQLite app batch backup\n homelab-auto-backup.service loaded inactive dead Homelab automatic full backup with retention\n homelab-backup-coverage-matrix.service loaded inactive dead Homelab backup coverage matrix\n homelab-backup-freshness.service loaded inactive dead Homelab backup freshness collector\n homelab-backup-healthcheck.service loaded inactive dead Homelab backup healthcheck\n● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\n homelab-backup-sla-health.service loaded inactive dead Homelab backup freshness SLA health\n homelab-beszel-backup.service loaded inactive dead Beszel encrypted two-cloud application backup\n homelab-blackbox-exporter-backup.service loaded inactive dead Blackbox Exporter encrypted two-cloud application backup\n homelab-cluster-admin-incident-engine-backup.service loaded inactive dead Cluster Admin Incident Engine encrypted two-cloud backup\n homelab-crypto-portfolio-backup.service loaded inactive dead Gram Crypto portfolio two-cloud backup\n homelab-emergency-backup.service loaded inactive dead Homelab emergency/config backup to two encrypted clouds\n homelab-extended-appbackup.service loaded inactive dead Homelab extended P1/P2 appbackup run\n homelab-gitea-secondary-backup.service loaded inactive dead Create and copy a verified Gitea backup to pve03\n homelab-legacy-monitoring-backups.service loaded inactive dead Legacy monitoring applications encrypted two-cloud backup batch\n homelab-mail-cloud-critical-upload.service loaded inactive dead Homelab Mail Cloud critical/small baseline upload\n homelab-mail-cloud-edge-vm-upload.service loaded inactive dead Homelab Mail Cloud edge-vm large chunked upload\n homelab-mail-cloud-nextcloud-vm-upload.service loaded inactive dead Homelab Mail Cloud Nextcloud VM vzdump upload\n homelab-mail-cloud-restore-drill.service loaded inactive dead Homelab Mail Cloud restore drill\n homelab-router-running-config-mail-cloud.service loaded inactive dead Homelab router running-config Mail-cloud backup\n homelab-verified-backup-health.service loaded inactive dead Homelab verified backup health evidence\n homelab-vm-backup-health.service loaded inactive dead Homelab VM backup health\n homelab-vm-backup-policy.service loaded inactive dead Homelab VM backup policy and feasibility health\n homelab-vm-full-backup.service loaded inactive dead Full vzdump backup of VM170 and VM171 to offhost and two clouds\n homelab-vm-local-dumps-2cloud.service loaded inactive dead Homelab VM/CT existing local dumps two-cloud verifier\n homelab-vm-local-dumps-retention.service loaded inactive dead Homelab VM local dumps cloud retention\n● homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\n immich-media-backup.service loaded inactive dead Immich media backup to staging\n immich-media-offhost-sync.service loaded inactive dead Sync Immich media backups from pve01 to pve02\n memos-backup.service loaded inactive dead Memos data backup to pve01 staging\n memos-offhost-sync.service loaded inactive dead Memos off-host backup sync to pve02\n postfix.service loaded active running Postfix Mail Transport Agent (main/default instance)\n postfix@-.service loaded inactive dead Postfix Mail Transport Agent (instance -, /etc/postfix-/)\n skladchik-reports-monitor-backup-restore-check.service loaded inactive dead Restore-check Skladchik reports monitor cloud backups\n skladchik-reports-monitor-backup.service loaded inactive dead Backup Skladchik reports monitor from edge-vm to encrypted clouds\n dpkg-db-backup.timer loaded active waiting Daily dpkg database backup timer\n filebrowser-backup.timer loaded active waiting Run File Browser backup\n homelab-alertmanager-backup.timer loaded active waiting Daily Alertmanager two-cloud backup\n homelab-appbackup-dockge-stacks.timer loaded active waiting Daily homelab app-level backup timer for dockge-stacks\n homelab-appbackup-mariadb-batch.timer loaded active waiting Daily Homelab MariaDB app batch backup timer\n homelab-appbackup-npmplus-edge.timer loaded active waiting Daily homelab app-level backup timer for npmplus-edge\n homelab-appbackup-postgres-batch.timer loaded active waiting Daily Homelab Postgres app batch backup timer\n homelab-appbackup-sqlite-batch.timer loaded active waiting Daily Homelab SQLite app batch backup timer\n homelab-auto-backup.timer loaded active waiting Run homelab automatic full backup weekly\n homelab-backup-coverage-matrix.timer loaded active waiting Homelab backup coverage matrix timer\n homelab-backup-freshness.timer loaded active waiting Run homelab backup freshness collector daily\n homelab-backup-healthcheck.timer loaded active waiting Run homelab backup healthcheck daily\n homelab-backup-sla-health.timer loaded active waiting Homelab backup freshness SLA health timer\n homelab-beszel-backup.timer loaded active waiting Daily Beszel encrypted two-cloud backup\n homelab-blackbox-exporter-backup.timer loaded active waiting Daily Blackbox Exporter encrypted two-cloud backup\n homelab-cluster-admin-incident-engine-backup.timer loaded active waiting Daily Cluster Admin Incident Engine two-cloud backup\n homelab-crypto-portfolio-backup.timer loaded active waiting Daily Gram Crypto portfolio backup\n homelab-emergency-backup.timer loaded active waiting Daily homelab emergency/config backup timer\n homelab-extended-appbackup.timer loaded active waiting Homelab extended P1/P2 appbackup timer\n homelab-gitea-secondary-backup.timer loaded active waiting Daily verified Gitea backup to pve03\n homelab-legacy-monitoring-backups.timer loaded active waiting Daily legacy monitoring applications two-cloud backup batch\n homelab-mail-cloud-critical-upload.timer loaded active waiting Daily Homelab Mail Cloud critical/small baseline upload\n homelab-mail-cloud-edge-vm-upload.timer loaded active waiting Weekly Homelab Mail Cloud edge-vm large chunked upload\n homelab-mail-cloud-nextcloud-vm-upload.timer loaded active waiting Run Homelab Mail Cloud Nextcloud VM upload daily\n homelab-mail-cloud-restore-drill.timer loaded active waiting Weekly Homelab Mail Cloud restore drill\n homelab-router-running-config-mail-cloud.timer loaded active waiting Run Homelab router running-config Mail-cloud backup daily\n homelab-verified-backup-health.timer loaded active waiting Homelab verified backup health evidence timer\n homelab-vm-backup-health.timer loaded active waiting Homelab VM backup health timer\n homelab-vm-backup-policy.timer loaded active waiting Homelab VM backup policy and feasibility health timer\n homelab-vm-full-backup.timer loaded active waiting Daily full backup timer for VM170 and VM171\n homelab-vm-local-dumps-2cloud.timer loaded active waiting Homelab VM/CT existing local dumps two-cloud verifier timer\n homelab-vm-local-dumps-retention.timer loaded active waiting Homelab VM local dumps cloud retention timer\n memos-backup.timer loaded active waiting Run Memos data backup daily\n memos-offhost-sync.timer loaded active waiting Run Memos off-host backup sync daily\n skladchik-reports-monitor-backup-restore-check.timer loaded active waiting Weekly restore-check of Skladchik reports monitor cloud backups\n skladchik-reports-monitor-backup.timer loaded active waiting Daily Skladchik reports monitor backup to encrypted clouds\nWed 2026-07-22 00:00:00 MSK 6h Tue 2026-07-21 00:00:14 MSK 17h ago dpkg-db-backup.timer dpkg-db-backup.service\nWed 2026-07-22 00:09:43 MSK 6h Tue 2026-07-21 00:09:32 MSK 17h ago homelab-backup-freshness.timer homelab-backup-freshness.service\nWed 2026-07-22 02:15:00 MSK 8h Tue 2026-07-21 02:15:05 MSK 15h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service\nWed 2026-07-22 03:10:02 MSK 9h Tue 2026-07-21 03:13:14 MSK 14h ago homelab-emergency-backup.timer homelab-emergency-backup.service\nWed 2026-07-22 03:35:00 MSK 9h Tue 2026-07-21 03:35:14 MSK 14h ago filebrowser-backup.timer filebrowser-backup.service\nWed 2026-07-22 03:55:02 MSK 10h Tue 2026-07-21 03:51:25 MSK 13h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service\nWed 2026-07-22 04:17:29 MSK 10h Tue 2026-07-21 04:19:17 MSK 13h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service\nWed 2026-07-22 04:26:54 MSK 10h Tue 2026-07-21 04:28:14 MSK 13h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service\nWed 2026-07-22 04:38:04 MSK 10h Tue 2026-07-21 04:34:48 MSK 13h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service\nWed 2026-07-22 05:01:44 MSK 11h Tue 2026-07-21 04:57:14 MSK 12h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service\nWed 2026-07-22 05:23:56 MSK 11h Tue 2026-07-21 05:17:57 MSK 12h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service\nWed 2026-07-22 05:27:13 MSK 11h Tue 2026-07-21 05:29:05 MSK 12h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service\nWed 2026-07-22 05:47:19 MSK 12h Tue 2026-07-21 05:46:14 MSK 11h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service\nWed 2026-07-22 06:01:19 MSK 12h Tue 2026-07-21 05:57:30 MSK 11h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service\nWed 2026-07-22 06:27:43 MSK 12h Tue 2026-07-21 06:32:25 MSK 11h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service\nWed 2026-07-22 06:38:51 MSK 12h Tue 2026-07-21 06:43:54 MSK 10h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nWed 2026-07-22 06:39:22 MSK 12h Tue 2026-07-21 06:40:25 MSK 10h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service\nWed 2026-07-22 06:44:16 MSK 13h Tue 2026-07-21 06:40:25 MSK 10h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nWed 2026-07-22 06:52:06 MSK 13h Tue 2026-07-21 06:47:47 MSK 10h ago homelab-beszel-backup.timer homelab-beszel-backup.service\nWed 2026-07-22 06:58:58 MSK 13h Tue 2026-07-21 06:57:30 MSK 10h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service\nWed 2026-07-22 07:02:27 MSK 13h Tue 2026-07-21 07:05:30 MSK 10h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service\nWed 2026-07-22 07:07:14 MSK 13h Tue 2026-07-21 07:18:55 MSK 10h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service\nWed 2026-07-22 07:35:06 MSK 13h Tue 2026-07-21 07:36:59 MSK 10h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service\nWed 2026-07-22 07:38:21 MSK 13h Tue 2026-07-21 07:36:14 MSK 10h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service\nWed 2026-07-22 07:41:49 MSK 14h Tue 2026-07-21 07:53:41 MSK 9h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nWed 2026-07-22 08:10:37 MSK 14h Tue 2026-07-21 08:07:25 MSK 9h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nWed 2026-07-22 08:37:02 MSK 14h Tue 2026-07-21 08:30:07 MSK 9h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service\nWed 2026-07-22 08:40:13 MSK 15h Tue 2026-07-21 08:50:14 MSK 8h ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service\nWed 2026-07-22 08:53:36 MSK 15h Tue 2026-07-21 08:41:14 MSK 8h ago memos-backup.timer memos-backup.service\nSun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service\nSun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service\nSun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 10h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\nJul 19 20:28:30 pve01 homelab-backup-matrix-refresh[4070223]: MATRIX_STATE_STATUS=FAIL\nJul 19 20:28:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 20:28:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 20:33:30 pve01 homelab-backup-matrix-refresh[4092638]: MATRIX_STATE_STATUS=FAIL\nJul 19 20:33:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 20:33:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 20:38:36 pve01 homelab-backup-matrix-refresh[4106577]: MATRIX_STATE_STATUS=FAIL\nJul 19 20:38:36 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 20:38:36 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 20:43:56 pve01 homelab-backup-matrix-refresh[4109241]: MATRIX_STATE_STATUS=FAIL\nJul 19 20:43:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 20:43:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 20:49:15 pve01 homelab-backup-matrix-refresh[4131689]: MATRIX_STATE_STATUS=FAIL\nJul 19 20:49:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 20:49:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 20:54:15 pve01 homelab-backup-matrix-refresh[4145509]: MATRIX_STATE_STATUS=FAIL\nJul 19 20:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 20:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 20:59:15 pve01 homelab-backup-matrix-refresh[4159482]: MATRIX_STATE_STATUS=FAIL\nJul 19 20:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 20:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:04:15 pve01 homelab-backup-matrix-refresh[4181782]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:09:15 pve01 homelab-backup-matrix-refresh[1993]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:14:15 pve01 homelab-backup-matrix-refresh[16108]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:19:15 pve01 homelab-backup-matrix-refresh[38546]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:24:15 pve01 homelab-backup-matrix-refresh[52499]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:29:15 pve01 homelab-backup-matrix-refresh[66476]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:34:15 pve01 homelab-backup-matrix-refresh[88886]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:39:15 pve01 homelab-backup-matrix-refresh[102884]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:44:15 pve01 homelab-backup-matrix-refresh[116812]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:49:26 pve01 homelab-backup-matrix-refresh[139246]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:49:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:49:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:54:30 pve01 homelab-backup-matrix-refresh[153291]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 21:59:30 pve01 homelab-backup-matrix-refresh[167280]: MATRIX_STATE_STATUS=FAIL\nJul 19 21:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 21:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:04:30 pve01 homelab-backup-matrix-refresh[189663]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:09:30 pve01 homelab-backup-matrix-refresh[192354]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:14:30 pve01 homelab-backup-matrix-refresh[206349]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:19:30 pve01 homelab-backup-matrix-refresh[228779]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:24:30 pve01 homelab-backup-matrix-refresh[242779]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:29:30 pve01 homelab-backup-matrix-refresh[256597]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:34:30 pve01 homelab-backup-matrix-refresh[279054]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:34:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:34:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:39:30 pve01 homelab-backup-matrix-refresh[293022]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:44:30 pve01 homelab-backup-matrix-refresh[307038]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:49:46 pve01 homelab-backup-matrix-refresh[330036]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:49:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:49:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 22:55:15 pve01 homelab-backup-matrix-refresh[345990]: MATRIX_STATE_STATUS=FAIL\nJul 19 22:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 22:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:00:16 pve01 homelab-backup-matrix-refresh[361536]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:05:15 pve01 homelab-backup-matrix-refresh[385062]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:10:15 pve01 homelab-backup-matrix-refresh[399332]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:15:16 pve01 homelab-backup-matrix-refresh[403605]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:20:15 pve01 homelab-backup-matrix-refresh[427739]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:25:15 pve01 homelab-backup-matrix-refresh[443219]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:30:16 pve01 homelab-backup-matrix-refresh[458795]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:35:15 pve01 homelab-backup-matrix-refresh[482814]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:40:15 pve01 homelab-backup-matrix-refresh[498238]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:45:16 pve01 homelab-backup-matrix-refresh[513836]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:50:26 pve01 homelab-backup-matrix-refresh[538010]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 19 23:55:30 pve01 homelab-backup-matrix-refresh[553603]: MATRIX_STATE_STATUS=FAIL\nJul 19 23:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 19 23:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:00:16 pve01 homelab-backup-matrix-refresh[569004]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:05:16 pve01 homelab-backup-matrix-refresh[593337]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:05:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:05:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:10:16 pve01 homelab-backup-matrix-refresh[609133]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:10:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:10:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:12:16 pve01 homelab-backup-matrix-refresh[610747]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:12:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:12:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:17:16 pve01 homelab-backup-matrix-refresh[635186]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:17:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:17:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:22:17 pve01 homelab-backup-matrix-refresh[650655]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:22:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:22:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:27:26 pve01 homelab-backup-matrix-refresh[666442]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:27:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:27:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:32:30 pve01 homelab-backup-matrix-refresh[690423]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:37:30 pve01 homelab-backup-matrix-refresh[706247]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:42:30 pve01 homelab-backup-matrix-refresh[721727]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:47:30 pve01 homelab-backup-matrix-refresh[745645]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:52:30 pve01 homelab-backup-matrix-refresh[761096]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 00:57:30 pve01 homelab-backup-matrix-refresh[776579]: MATRIX_STATE_STATUS=FAIL\nJul 20 00:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 00:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:02:30 pve01 homelab-backup-matrix-refresh[797728]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:07:30 pve01 homelab-backup-matrix-refresh[804631]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:12:30 pve01 homelab-backup-matrix-refresh[820114]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:17:30 pve01 homelab-backup-matrix-refresh[844105]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:22:30 pve01 homelab-backup-matrix-refresh[859499]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:27:30 pve01 homelab-backup-matrix-refresh[875036]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:32:30 pve01 homelab-backup-matrix-refresh[899054]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:37:30 pve01 homelab-backup-matrix-refresh[914581]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:42:30 pve01 homelab-backup-matrix-refresh[930107]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:47:30 pve01 homelab-backup-matrix-refresh[954079]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:52:56 pve01 homelab-backup-matrix-refresh[969872]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 01:58:08 pve01 homelab-backup-matrix-refresh[985563]: MATRIX_STATE_STATUS=FAIL\nJul 20 01:58:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 01:58:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:03:15 pve01 homelab-backup-matrix-refresh[1009625]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:08:15 pve01 homelab-backup-matrix-refresh[1025172]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:13:15 pve01 homelab-backup-matrix-refresh[1040687]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:13:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:13:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:18:15 pve01 homelab-backup-matrix-refresh[1064737]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:18:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:18:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:23:15 pve01 homelab-backup-matrix-refresh[1080263]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:23:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:23:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:28:16 pve01 homelab-backup-matrix-refresh[1084502]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:28:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:28:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:33:15 pve01 homelab-backup-matrix-refresh[1108551]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:38:15 pve01 homelab-backup-matrix-refresh[1124188]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:43:15 pve01 homelab-backup-matrix-refresh[1139799]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:48:15 pve01 homelab-backup-matrix-refresh[1163841]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:53:15 pve01 homelab-backup-matrix-refresh[1179474]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:53:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:53:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 02:58:16 pve01 homelab-backup-matrix-refresh[1195011]: MATRIX_STATE_STATUS=FAIL\nJul 20 02:58:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 02:58:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:03:16 pve01 homelab-backup-matrix-refresh[1219515]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:03:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:03:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:08:17 pve01 homelab-backup-matrix-refresh[1235611]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:08:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:08:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:09:51 pve01 homelab-backup-matrix-refresh[1237636]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:09:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:09:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:13:42 pve01 homelab-backup-matrix-refresh[1252592]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:13:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:13:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:19:06 pve01 homelab-backup-matrix-refresh[1276835]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:19:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:19:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:24:07 pve01 homelab-backup-matrix-refresh[1292418]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:29:15 pve01 homelab-backup-matrix-refresh[1307915]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:30:00 pve01 pvescheduler[1308929]: INFO: starting new backup job: vzdump 110 111 112 113 130 150 190 --prune-backups 'keep-daily=4' --mode snapshot --mailnotification failure --storage local --compress zstd --quiet 1\nJul 20 03:34:15 pve01 homelab-backup-matrix-refresh[1332038]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:35:08 pve01 homelab-backup-matrix-refresh[1333178]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:35:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:35:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:40:15 pve01 homelab-backup-matrix-refresh[1348754]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:45:16 pve01 homelab-backup-matrix-refresh[1364258]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:50:16 pve01 homelab-backup-matrix-refresh[1388202]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:52:56 pve01 homelab-backup-matrix-refresh[1390999]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 03:58:15 pve01 homelab-backup-matrix-refresh[1406759]: MATRIX_STATE_STATUS=FAIL\nJul 20 03:58:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 03:58:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:03:15 pve01 homelab-backup-matrix-refresh[1430825]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:08:15 pve01 homelab-backup-matrix-refresh[1446346]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:10:15 pve01 homelab-backup-matrix-refresh[1448294]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 20 04:15:26 pve01 homelab-backup-matrix-refresh[1464728]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:15:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:15:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:20:01 pve01 homelab-backup-matrix-refresh[1488281]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:20:01 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:20:01 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:25:12 pve01 homelab-backup-matrix-refresh[1503745]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:25:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:25:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:25:47 pve01 homelab-backup-matrix-refresh[1505286]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:25:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:25:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:31:12 pve01 homelab-backup-matrix-refresh[1529373]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:31:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:31:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:36:15 pve01 homelab-backup-matrix-refresh[1544983]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:41:15 pve01 homelab-backup-matrix-refresh[1560445]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:46:15 pve01 homelab-backup-matrix-refresh[1584307]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:51:15 pve01 homelab-backup-matrix-refresh[1599773]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 04:56:15 pve01 homelab-backup-matrix-refresh[1615263]: MATRIX_STATE_STATUS=FAIL\nJul 20 04:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 04:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:01:16 pve01 homelab-backup-matrix-refresh[1640460]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:01:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:01:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:03:43 pve01 homelab-backup-matrix-refresh[1644969]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:03:43 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:03:43 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:08:48 pve01 homelab-backup-matrix-refresh[1660506]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:08:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:08:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:13:56 pve01 homelab-backup-matrix-refresh[1675931]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:13:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:13:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:19:07 pve01 homelab-backup-matrix-refresh[1700241]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:19:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:19:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:24:07 pve01 homelab-backup-matrix-refresh[1715665]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:27:56 pve01 homelab-backup-matrix-refresh[1733709]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:27:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:27:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:33:15 pve01 homelab-backup-matrix-refresh[1757822]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:38:15 pve01 homelab-backup-matrix-refresh[1773361]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:43:15 pve01 homelab-backup-matrix-refresh[1788819]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:48:15 pve01 homelab-backup-matrix-refresh[1812619]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:49:50 pve01 homelab-backup-matrix-refresh[1814730]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:49:50 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:49:50 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 05:54:56 pve01 homelab-backup-matrix-refresh[1830204]: MATRIX_STATE_STATUS=FAIL\nJul 20 05:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 05:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:00:16 pve01 homelab-backup-matrix-refresh[1845897]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:05:15 pve01 homelab-backup-matrix-refresh[1872116]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:10:15 pve01 homelab-backup-matrix-refresh[1893683]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:13:16 pve01 homelab-backup-matrix-refresh[1900739]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:13:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:13:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:18:18 pve01 homelab-backup-matrix-refresh[1924679]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:18:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:18:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:23:26 pve01 homelab-backup-matrix-refresh[1940188]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:23:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:23:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:28:26 pve01 homelab-backup-matrix-refresh[1956242]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:28:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:28:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:33:26 pve01 homelab-backup-matrix-refresh[1980145]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:33:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:33:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:35:21 pve01 homelab-backup-matrix-refresh[1984457]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:35:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:35:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:40:26 pve01 homelab-backup-matrix-refresh[2000528]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:45:30 pve01 homelab-backup-matrix-refresh[2016958]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:50:30 pve01 homelab-backup-matrix-refresh[2040093]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 06:52:20 pve01 homelab-mail-cloud-critical-upload[2039699]: REMOTE_FAIL=0\nJul 20 06:55:30 pve01 homelab-backup-matrix-refresh[2056859]: MATRIX_STATE_STATUS=FAIL\nJul 20 06:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 06:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:00:30 pve01 homelab-backup-matrix-refresh[2073835]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:05:30 pve01 homelab-backup-matrix-refresh[2097030]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:08:29 pve01 homelab-backup-matrix-refresh[2111023]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:08:29 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:08:29 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:13:30 pve01 homelab-backup-matrix-refresh[2123578]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:13:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:13:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:18:30 pve01 homelab-backup-matrix-refresh[2140101]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:18:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:18:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:23:30 pve01 homelab-backup-matrix-refresh[2158207]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:23:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:23:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:25:15 pve01 homelab-backup-matrix-refresh[2171082]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:30:16 pve01 homelab-backup-matrix-refresh[2186510]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:35:15 pve01 homelab-backup-matrix-refresh[2210494]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:40:21 pve01 homelab-backup-matrix-refresh[2226665]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:40:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:40:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:41:30 pve01 homelab-backup-matrix-refresh[2227922]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:46:32 pve01 homelab-backup-matrix-refresh[2251943]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:46:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:46:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:51:56 pve01 homelab-backup-matrix-refresh[2267857]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:51:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:51:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 07:57:08 pve01 homelab-backup-matrix-refresh[2284427]: MATRIX_STATE_STATUS=FAIL\nJul 20 07:57:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 07:57:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:02:16 pve01 homelab-backup-matrix-refresh[2297063]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:02:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:02:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:07:15 pve01 homelab-backup-matrix-refresh[2316882]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:10:25 pve01 homelab-overall-health[2331217]: STATUS=WARN TS=2026-07-20T05:10:25Z TYPE=overall-health BAD=5 backup-framework.txt=OK drift-check.txt=WARN service-registry.txt=OK dependency-map.txt=OK golden-state.txt=OK cluster-passport.txt=WARN final-readiness.txt=WARN safe-autoheal.txt=OK kuma-monitor-policy.txt=OK backup-sla.txt=OK backup-coverage-matrix.txt=OK extended-appbackup.txt=OK residual-review.txt=OK forum-snuffleupagus.txt=OK incident-journal.txt=OK duty-admin-v2.txt=OK node-loss-readiness.txt=OK node-loss-runbooks.txt=OK power-loss-readiness.txt=OK power-loss-runbook.txt=OK ungated-health-backlog.txt=OK pve03-root-cleanup.txt=OK pve03-staging-retention.txt=OK pve03-staging-retention-proof.txt=OK pve03-staging-retention-cleanup.txt=OK pve03-failed-unit-cleanup.txt=OK remote-git-sops-age-readiness.txt=OK remote-git-sops-age-policy.txt=OK desired-state-remote-target-trace.txt=OK desired-state-remote-target.txt=OK desired-state.txt=OK runbooks.txt=OK alerting.txt=OK secret-exposure.txt=OK capacity-risk.txt=OK vm-local-dumps-retention.txt=OK vm-local-dumps-cloud.txt=OK vm-backup-policy.txt=OK vm170-vm171-full-strategy.txt=OK external-probe-vps.txt=OK netbird-vps-trust.txt=OK netbird-vps-trust-closure.txt=OK external-probe-runner-decision.txt=OK netbird-vps-trust-clean-seal.txt=OK live-tail-audit.txt=OK vm-backup.txt=OK cluster-admin-observer.txt=OK cluster-admin-restricted-probes.txt=WARN cluster-admin-full-observer.txt=WARN cluster-admin-vm-mail-cloud-backup.txt=OK cluster-admin-webpanel-sync.txt=OK cluster-admin-webpanel.txt=OK\nJul 20 08:10:27 pve01 homelab-backup-matrix-refresh[2331605]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:10:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:10:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:15:30 pve01 homelab-backup-matrix-refresh[2347983]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:20:30 pve01 homelab-backup-matrix-refresh[2371091]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:25:30 pve01 homelab-backup-matrix-refresh[2386601]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:30:22 pve01 homelab-backup-matrix-refresh[2403498]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:30:22 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:30:22 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:35:26 pve01 homelab-backup-matrix-refresh[2426661]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:35:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:35:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:36:16 pve01 homelab-backup-matrix-refresh[2427543]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:36:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:36:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:41:18 pve01 homelab-backup-matrix-refresh[2443027]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:41:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:41:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:46:26 pve01 homelab-backup-matrix-refresh[2467111]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:46:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:46:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:47:15 pve01 homelab-backup-matrix-refresh[2468144]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:47:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:47:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:52:26 pve01 homelab-backup-matrix-refresh[2484169]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:52:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:52:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 08:57:30 pve01 homelab-backup-matrix-refresh[2499762]: MATRIX_STATE_STATUS=FAIL\nJul 20 08:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 08:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:02:30 pve01 homelab-backup-matrix-refresh[2523793]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:07:30 pve01 homelab-backup-matrix-refresh[2539245]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:12:30 pve01 homelab-backup-matrix-refresh[2554758]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:17:30 pve01 homelab-backup-matrix-refresh[2567370]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:22:30 pve01 homelab-backup-matrix-refresh[2583117]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:27:30 pve01 homelab-backup-matrix-refresh[2598614]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:32:30 pve01 homelab-backup-matrix-refresh[2622549]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:37:30 pve01 homelab-backup-matrix-refresh[2638103]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:42:30 pve01 homelab-backup-matrix-refresh[2653680]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:47:30 pve01 homelab-backup-matrix-refresh[2677702]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:52:30 pve01 homelab-backup-matrix-refresh[2693207]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 09:57:30 pve01 homelab-backup-matrix-refresh[2708789]: MATRIX_STATE_STATUS=FAIL\nJul 20 09:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 09:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:02:30 pve01 homelab-backup-matrix-refresh[2732734]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:04:07 pve01 homelab-backup-matrix-refresh[2734125]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:04:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:04:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:09:07 pve01 homelab-backup-matrix-refresh[2749537]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:09:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:09:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:14:07 pve01 homelab-backup-matrix-refresh[2765091]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:14:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:14:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:19:15 pve01 homelab-backup-matrix-refresh[2789006]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:24:15 pve01 homelab-backup-matrix-refresh[2804462]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:29:15 pve01 homelab-backup-matrix-refresh[2819907]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:34:15 pve01 homelab-backup-matrix-refresh[2843900]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:39:15 pve01 homelab-backup-matrix-refresh[2859366]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:44:16 pve01 homelab-backup-matrix-refresh[2863481]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:44:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:44:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:49:15 pve01 homelab-backup-matrix-refresh[2887434]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:54:15 pve01 homelab-backup-matrix-refresh[2902771]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 10:59:15 pve01 homelab-backup-matrix-refresh[2918259]: MATRIX_STATE_STATUS=FAIL\nJul 20 10:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 10:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:04:15 pve01 homelab-backup-matrix-refresh[2942234]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:09:15 pve01 homelab-backup-matrix-refresh[2957726]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:14:15 pve01 homelab-backup-matrix-refresh[2973175]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:19:15 pve01 homelab-backup-matrix-refresh[2997114]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:24:15 pve01 homelab-backup-matrix-refresh[3012539]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:29:15 pve01 homelab-backup-matrix-refresh[3028098]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:34:26 pve01 homelab-backup-matrix-refresh[3052177]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:34:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:34:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:39:30 pve01 homelab-backup-matrix-refresh[3067593]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:44:30 pve01 homelab-backup-matrix-refresh[3083139]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:49:30 pve01 homelab-backup-matrix-refresh[3107024]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:49:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:49:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:54:30 pve01 homelab-backup-matrix-refresh[3122507]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 11:59:30 pve01 homelab-backup-matrix-refresh[3137990]: MATRIX_STATE_STATUS=FAIL\nJul 20 11:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 11:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:04:30 pve01 homelab-backup-matrix-refresh[3150598]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:09:30 pve01 homelab-backup-matrix-refresh[3166051]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:14:30 pve01 homelab-backup-matrix-refresh[3181454]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:19:30 pve01 homelab-backup-matrix-refresh[3205349]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:24:30 pve01 homelab-backup-matrix-refresh[3220920]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:29:30 pve01 homelab-backup-matrix-refresh[3236455]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:34:56 pve01 homelab-backup-matrix-refresh[3260683]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:34:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:34:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:40:08 pve01 homelab-backup-matrix-refresh[3276379]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:40:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:40:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:45:09 pve01 homelab-backup-matrix-refresh[3297387]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:45:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:45:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:50:15 pve01 homelab-backup-matrix-refresh[3315975]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 12:55:15 pve01 homelab-backup-matrix-refresh[3331521]: MATRIX_STATE_STATUS=FAIL\nJul 20 12:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 12:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:00:15 pve01 homelab-backup-matrix-refresh[3347832]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:05:15 pve01 homelab-backup-matrix-refresh[3370899]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:10:15 pve01 homelab-backup-matrix-refresh[3386358]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:15:16 pve01 homelab-backup-matrix-refresh[3401894]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:20:15 pve01 homelab-backup-matrix-refresh[3425830]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:25:15 pve01 homelab-backup-matrix-refresh[3429905]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:30:15 pve01 homelab-backup-matrix-refresh[3446239]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:30:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:30:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:35:15 pve01 homelab-backup-matrix-refresh[3469372]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:40:15 pve01 homelab-backup-matrix-refresh[3484818]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:45:16 pve01 homelab-backup-matrix-refresh[3500319]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:50:26 pve01 homelab-backup-matrix-refresh[3524417]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 13:55:30 pve01 homelab-backup-matrix-refresh[3539958]: MATRIX_STATE_STATUS=FAIL\nJul 20 13:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 13:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:00:30 pve01 homelab-backup-matrix-refresh[3556311]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:05:30 pve01 homelab-backup-matrix-refresh[3579422]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:10:30 pve01 homelab-backup-matrix-refresh[3594909]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:10:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:10:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:15:30 pve01 homelab-backup-matrix-refresh[3618728]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:20:30 pve01 homelab-backup-matrix-refresh[3634183]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:25:30 pve01 homelab-backup-matrix-refresh[3649674]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:30:30 pve01 homelab-backup-matrix-refresh[3673515]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:30:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:30:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:35:30 pve01 homelab-backup-matrix-refresh[3689095]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:35:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:35:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:40:31 pve01 homelab-backup-matrix-refresh[3698580]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:45:30 pve01 homelab-backup-matrix-refresh[3709574]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:50:30 pve01 homelab-backup-matrix-refresh[3732500]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 14:55:30 pve01 homelab-backup-matrix-refresh[3747946]: MATRIX_STATE_STATUS=FAIL\nJul 20 14:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 14:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:00:30 pve01 homelab-backup-matrix-refresh[3771782]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:05:48 pve01 homelab-backup-matrix-refresh[3787552]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:05:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:05:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:11:06 pve01 homelab-backup-matrix-refresh[3803194]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:11:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:11:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:16:16 pve01 homelab-backup-matrix-refresh[3819582]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:16:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:16:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:21:15 pve01 homelab-backup-matrix-refresh[3842674]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:26:15 pve01 homelab-backup-matrix-refresh[3858134]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:26:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:26:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:31:15 pve01 homelab-backup-matrix-refresh[3874469]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:36:15 pve01 homelab-backup-matrix-refresh[3897548]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:41:15 pve01 homelab-backup-matrix-refresh[3912924]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:46:16 pve01 homelab-backup-matrix-refresh[3917964]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:46:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:46:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:51:15 pve01 homelab-backup-matrix-refresh[3941009]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 15:56:15 pve01 homelab-backup-matrix-refresh[3956553]: MATRIX_STATE_STATUS=FAIL\nJul 20 15:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 15:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:01:15 pve01 homelab-backup-matrix-refresh[3972944]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:01:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:01:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:06:15 pve01 homelab-backup-matrix-refresh[3996047]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:06:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:06:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:11:15 pve01 homelab-backup-matrix-refresh[4011569]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:16:15 pve01 homelab-backup-matrix-refresh[4035432]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:16:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:16:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:21:15 pve01 homelab-backup-matrix-refresh[4050883]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:26:16 pve01 homelab-backup-matrix-refresh[4066337]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:26:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:26:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:31:18 pve01 homelab-backup-matrix-refresh[4090328]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:31:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:31:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:36:26 pve01 homelab-backup-matrix-refresh[4106097]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:36:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:36:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:41:30 pve01 homelab-backup-matrix-refresh[4121654]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:46:30 pve01 homelab-backup-matrix-refresh[4145630]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:51:30 pve01 homelab-backup-matrix-refresh[4161061]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 16:56:31 pve01 homelab-backup-matrix-refresh[4176634]: MATRIX_STATE_STATUS=FAIL\nJul 20 16:56:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 16:56:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:01:30 pve01 homelab-backup-matrix-refresh[6881]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:06:30 pve01 homelab-backup-matrix-refresh[22575]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:11:30 pve01 homelab-backup-matrix-refresh[38083]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:16:31 pve01 homelab-backup-matrix-refresh[59658]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:21:30 pve01 homelab-backup-matrix-refresh[66183]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:26:30 pve01 homelab-backup-matrix-refresh[81598]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:31:31 pve01 homelab-backup-matrix-refresh[105617]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:36:30 pve01 homelab-backup-matrix-refresh[121433]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:36:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:36:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:41:30 pve01 homelab-backup-matrix-refresh[136937]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:46:30 pve01 homelab-backup-matrix-refresh[160897]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:51:30 pve01 homelab-backup-matrix-refresh[176488]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 17:56:30 pve01 homelab-backup-matrix-refresh[191911]: MATRIX_STATE_STATUS=FAIL\nJul 20 17:56:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 17:56:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:01:30 pve01 homelab-backup-matrix-refresh[215914]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:06:30 pve01 homelab-backup-matrix-refresh[231465]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:11:30 pve01 homelab-backup-matrix-refresh[247009]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:16:31 pve01 homelab-backup-matrix-refresh[270971]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:21:30 pve01 homelab-backup-matrix-refresh[286516]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:26:30 pve01 homelab-backup-matrix-refresh[290662]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:31:30 pve01 homelab-backup-matrix-refresh[314846]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:36:46 pve01 homelab-backup-matrix-refresh[330621]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:36:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:36:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:41:48 pve01 homelab-backup-matrix-refresh[346168]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:41:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:41:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:46:58 pve01 homelab-backup-matrix-refresh[370235]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:46:58 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:46:58 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:52:08 pve01 homelab-backup-matrix-refresh[385867]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:52:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:52:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 18:57:15 pve01 homelab-backup-matrix-refresh[401443]: MATRIX_STATE_STATUS=FAIL\nJul 20 18:57:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 18:57:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:02:15 pve01 homelab-backup-matrix-refresh[425446]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:02:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:02:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:07:15 pve01 homelab-backup-matrix-refresh[440954]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:12:15 pve01 homelab-backup-matrix-refresh[456367]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:12:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:12:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:17:15 pve01 homelab-backup-matrix-refresh[480328]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:17:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:17:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:22:15 pve01 homelab-backup-matrix-refresh[495871]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:22:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:22:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:27:15 pve01 homelab-backup-matrix-refresh[511358]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:27:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:27:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:28:51 pve01 homelab-backup-matrix-refresh[512760]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:28:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:28:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.\nJul 20 19:34:08 pve01 homelab-backup-matrix-refresh[537480]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:34:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:34:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:39:09 pve01 homelab-backup-matrix-refresh[553115]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:39:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:39:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:44:15 pve01 homelab-backup-matrix-refresh[568668]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:49:15 pve01 homelab-backup-matrix-refresh[592723]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:54:15 pve01 homelab-backup-matrix-refresh[608272]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 19:59:15 pve01 homelab-backup-matrix-refresh[623767]: MATRIX_STATE_STATUS=FAIL\nJul 20 19:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 19:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:04:15 pve01 homelab-backup-matrix-refresh[647775]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:09:15 pve01 homelab-backup-matrix-refresh[663153]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:14:15 pve01 homelab-backup-matrix-refresh[667315]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:19:26 pve01 homelab-backup-matrix-refresh[691333]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:19:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:19:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:24:31 pve01 homelab-backup-matrix-refresh[706901]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:24:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:24:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:29:31 pve01 homelab-backup-matrix-refresh[722368]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:29:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:29:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:34:31 pve01 homelab-backup-matrix-refresh[746391]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:34:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:34:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:39:31 pve01 homelab-backup-matrix-refresh[761801]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:39:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:39:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:44:31 pve01 homelab-backup-matrix-refresh[777273]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:44:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:44:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:49:31 pve01 homelab-backup-matrix-refresh[801193]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:49:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:49:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 20:54:56 pve01 homelab-backup-matrix-refresh[816954]: MATRIX_STATE_STATUS=FAIL\nJul 20 20:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 20:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:00:16 pve01 homelab-backup-matrix-refresh[832684]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:05:15 pve01 homelab-backup-matrix-refresh[856679]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:10:15 pve01 homelab-backup-matrix-refresh[872167]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:15:16 pve01 homelab-backup-matrix-refresh[887656]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:20:15 pve01 homelab-backup-matrix-refresh[911605]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:25:16 pve01 homelab-backup-matrix-refresh[915711]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:25:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:25:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:30:16 pve01 homelab-backup-matrix-refresh[931433]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:35:16 pve01 homelab-backup-matrix-refresh[955099]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:40:26 pve01 homelab-backup-matrix-refresh[970563]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:45:28 pve01 homelab-backup-matrix-refresh[986962]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:45:28 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:45:28 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:50:30 pve01 homelab-backup-matrix-refresh[1009969]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 21:55:30 pve01 homelab-backup-matrix-refresh[1025505]: MATRIX_STATE_STATUS=FAIL\nJul 20 21:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 21:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:00:31 pve01 homelab-backup-matrix-refresh[1041885]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:05:31 pve01 homelab-backup-matrix-refresh[1065096]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:10:31 pve01 homelab-backup-matrix-refresh[1080543]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:15:31 pve01 homelab-backup-matrix-refresh[1096922]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:15:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:15:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:20:31 pve01 homelab-backup-matrix-refresh[1120354]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:20:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:20:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:25:31 pve01 homelab-backup-matrix-refresh[1135777]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:25:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:25:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:30:31 pve01 homelab-backup-matrix-refresh[1152095]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:30:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:30:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:35:31 pve01 homelab-backup-matrix-refresh[1175243]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:35:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:35:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:40:31 pve01 homelab-backup-matrix-refresh[1179415]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:45:31 pve01 homelab-backup-matrix-refresh[1195757]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:45:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:45:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:50:31 pve01 homelab-backup-matrix-refresh[1218911]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:50:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:50:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 22:55:31 pve01 homelab-backup-matrix-refresh[1234331]: MATRIX_STATE_STATUS=FAIL\nJul 20 22:55:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 22:55:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:00:31 pve01 homelab-backup-matrix-refresh[1258173]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:05:31 pve01 homelab-backup-matrix-refresh[1273772]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:10:31 pve01 homelab-backup-matrix-refresh[1289212]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:15:32 pve01 homelab-backup-matrix-refresh[1313105]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:15:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:15:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:20:48 pve01 homelab-backup-matrix-refresh[1328836]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:20:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:20:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:25:49 pve01 homelab-backup-matrix-refresh[1344339]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:25:49 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:25:49 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:31:15 pve01 homelab-backup-matrix-refresh[1360964]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:36:15 pve01 homelab-backup-matrix-refresh[1384068]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:41:15 pve01 homelab-backup-matrix-refresh[1399485]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:46:15 pve01 homelab-backup-matrix-refresh[1423238]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:51:16 pve01 homelab-backup-matrix-refresh[1427834]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:51:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:51:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 20 23:56:15 pve01 homelab-backup-matrix-refresh[1443462]: MATRIX_STATE_STATUS=FAIL\nJul 20 23:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 20 23:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:00:16 pve01 homelab-backup-matrix-refresh[1458364]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:05:26 pve01 homelab-backup-matrix-refresh[1482896]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:05:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:05:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:09:33 pve01 homelab-backup-matrix-refresh[1498014]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:09:33 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:09:33 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:09:34 pve01 homelab-safe-exec[1498016]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\nJul 21 00:10:55 pve01 homelab-safe-exec[1499369]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nJul 21 00:10:55 pve01 homelab-safe-exec[1499369]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\nJul 21 00:14:48 pve01 homelab-backup-matrix-refresh[1513957]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:14:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:14:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:20:00 pve01 homelab-backup-matrix-refresh[1538116]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:20:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:20:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:25:04 pve01 homelab-backup-matrix-refresh[1553654]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:25:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:25:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:26:08 pve01 homelab-safe-exec[1554627]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nJul 21 00:26:08 pve01 homelab-safe-exec[1554627]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\nJul 21 00:30:05 pve01 homelab-backup-matrix-refresh[1569592]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:30:05 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:30:05 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:35:16 pve01 homelab-backup-matrix-refresh[1594059]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:40:15 pve01 homelab-backup-matrix-refresh[1609512]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:42:17 pve01 homelab-safe-exec[1611283]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nJul 21 00:42:17 pve01 homelab-safe-exec[1611283]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\nJul 21 00:45:15 pve01 homelab-backup-matrix-refresh[1633504]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:45:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:45:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:50:15 pve01 homelab-backup-matrix-refresh[1637691]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:55:15 pve01 homelab-backup-matrix-refresh[1653171]: MATRIX_STATE_STATUS=FAIL\nJul 21 00:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 00:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 00:58:17 pve01 homelab-safe-exec[1667105]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nJul 21 00:58:17 pve01 homelab-safe-exec[1667105]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\nJul 21 01:00:15 pve01 homelab-backup-matrix-refresh[1669525]: MATRIX_STATE_STATUS=FAIL\nJul 21 01:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nJul 21 01:05:15 pve01 homelab-backup-matrix-refresh[1692700]: MATRIX_STATE_STATUS=FAIL\nJul 21 01:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nJul 21 01:05:15 pve01 systemd[1]: homelab-backup-matri" + "output": "=== IMMUTABLE_RESULT_VERIFIED ===\nCOMMAND_ID=CRITICAL-FIRST-DIAGNOSTIC-103\nIMMUTABLE_COMMIT=c4c3e49dae61e1904c196db23d9095d8f35f620c\ncut: '\"': No such file or directory\nJSON_SHA256=\ncut: '\"': No such file or directory\nTXT_SHA256=\n=== DECISION_COUNTS ===\nQUORUM_RISKS=0\nBACKUP_RISKS=1202\nKERNEL_RISKS=0\nFAILED_UNIT_RISKS=6\nCAPACITY_RISKS=0\n=== NEXT_ACTION ===\nPRIORITY=P0_BACKUP_POLICY\nEVIDENCE=● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nCRITICAL_DIAGNOSTIC_DECISION=COMPLETE\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index e643f7c..f579bfd 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,5 +1,5 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=CRITICAL-FIRST-DIAGNOSTIC-103 +COMMAND_ID=CRITICAL-DIAGNOSTIC-DECISION-104 STATUS=OK RC=0 HOST=pve01 @@ -9,2333 +9,33 @@ REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=32ad0722a84aebeb4f1e33dab07a771683458bec2422895b557411be769cd3f2 +COMMAND_SHA256=2d76a56126624be128d7ea0cc6526f032944f0e28b5fa7afa7f4a3a763b60835 DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=b23c2f575c56d419a6dfb673cbf42ce15bc9a9c8e2892d3d6a6f9a91c9a7a3ce -SANITIZED_OUTPUT_SHA256=2a82a08aa78353751e68781b74d3029143cd010fb8931ff0b43efc0b8686b853 +RAW_EVIDENCE_SHA256=8b71df5e3fea77b866119cdf7ce9025e569b8bb49ca9484943ad44465f53c5dc +SANITIZED_OUTPUT_SHA256=8b71df5e3fea77b866119cdf7ce9025e569b8bb49ca9484943ad44465f53c5dc OUTPUT_BEGIN -=== VERIFIED_PREVIOUS_RESULT === -COMMAND_ID=CRITICAL-FIRST-ACTION-102 -IMMUTABLE_COMMIT=4cf04bbfdae66e66e09e451a6a5eae4abb874704 +=== IMMUTABLE_RESULT_VERIFIED === +COMMAND_ID=CRITICAL-FIRST-DIAGNOSTIC-103 +IMMUTABLE_COMMIT=c4c3e49dae61e1904c196db23d9095d8f35f620c cut: '"': No such file or directory JSON_SHA256= cut: '"': No such file or directory TXT_SHA256= -PREVIOUS_COMMAND_IN_ERROR_REGISTER=no -=== FIRST_PRIORITY_FINDINGS === -=== FINDING_COUNTS === -P0_MATCHES=45 -P1_MATCHES=87 -STORAGE_RISK_MATCHES=2 -RECENT_FAILED_COMMAND_LINES=0 -BOOTSTRAP_SECTION_LINES=0 -=== ACTIONABLE_P0 === -=== ACTIONABLE_P0 === -86:398:- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained. -155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure. -203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (. -486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800 -508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000 -513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800 -546:● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -548:● homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -1102:/var/lib/homelab-health/backup-matrix-cluster-final-seal.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z -1104:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-production-closure.txt:1:STATUS=OK TS=2026-07-13T08:12:14Z TYPE=crypto-portfolio-ui-center-alert-currency-production-closure BAD=0 PRODUCTION_CHANGED=YES_UI_ONLY INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 POSTCHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION_ROLLBACK_COUNT=3 RETENTION_OPTIONAL_OTHER_CANDIDATES=1 RETENTION_REMOVED_ROLLBACK_COUNT=0 DANGLING_IMAGES_REMAINING=2 UNREFERENCED_DANGLING_IMAGES=0 DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE_CLEAN=YES PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES READY_FOR_UI_FINAL_SEAL=YES -1106:/var/lib/homelab-health/cluster-admin-stage3-postapply-backup.txt:1:STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z -1109:/var/lib/homelab-health/pve01-core-monitoring-app-restorecheck.txt:1:STATUS=OK TS=2026-07-08T15:57:27Z TYPE=pve01-core-monitoring-app-restorecheck BAD=0 MODE=APP_LEVEL_RESTORE_CHECKS_TWO_CLOUDS FULL_VM_RESTORE_AVAILABLE=NO TOTAL_APPS=30 OK_APPS=30 FAIL_LIST=none REPORT=/var/lib/homelab-node-loss/pve01-core-monitoring-app-level/app-restorecheck-latest.md -1118:/var/lib/homelab-health/hapusya-site-final-seal.txt:1:STATUS=OK TS=2026-07-13T12:33:08Z TYPE=hapusya-site-final-seal BAD=0 APP=hapusya-site URL=https://hapusya.ru/ PRESEAL_DESIRED_STATE_HEAD=69966eb83fc065141df3c46df0d6c0f282275046 ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb APPLY_MANIFEST_SHA256=f44992e3e0cc443b372779cb35d9e899d904a9960a9de03c14559886fdb1fc75 PRODUCTION_BROWSER_MANIFEST_SHA256=9f3cdf3063f8724d1d1c1ad619f47d6f98d434c3422a74903585146858a8dfac BROWSER_SMOKE=OK BROWSER_RUNS=5 BROWSER_ERRORS=0 DESKTOP_VIDEO=OK MOBILE_VIDEO=OK STATIC_FALLBACK=OK BACKUP_JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 TWO_CLOUDS=OK RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK DESIRED_STATE_SYNC=OK GENERATOR_SHA256=507bbbc2691f1f08cb1dd3660ed76cf13a8c36c23833d5fdec40cf6e07486cb8 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 POLICY_SHA256=4bf996ff451ae3ec56eb2fcd42842606cd97295b4d5141aedb135c956395e4b5 EVIDENCE_ROOT=/var/lib/homelab-change-sets/hapusya-site-final-seal/20260713T123308Z EVIDENCE_MANIFEST_SHA256=7eb993a17ac18a1f7c9ae0b52d745d3778833d15626ed34c82496f74916eb26a PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY DESIRED_STATE_CHANGED=YES_FINAL_SEAL_COMMIT_ONLY ALL_HAPUSYA_SITE_TAILS_CLOSED=YES TASK_COMPLETE=YES -1120:/var/lib/homelab-health/pve03-failed-unit-cleanup-closure.txt:1:STATUS=OK TS=2026-07-07T12:52:21Z TYPE=pve03-failed-unit-cleanup-closure BAD=0 WARN=0 MODE=READ_ONLY_CLOSURE LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CP_MATCH=YES RESTORE_CHECK=YES PVE03_FAILED_UNITS=0 REPORT=/var/lib/homelab-pve03-failed-unit-cleanup/closure-latest.md -1128:/var/lib/homelab-health/cluster-admin-backup-webpanel-final-seal.txt:1:STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[PRIVATE_IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md -1132:/var/lib/homelab-health/appbackupctl-routing.txt:1:STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z -1134:/var/lib/homelab-health/cluster-admin-stage3-api-apply.txt:1:STATUS=OK TS=2026-07-13T03:22:35Z TYPE=cluster-admin-stage3-api-apply BAD=0 COMMAND_PHYSICAL_LINES=1 RESULT=APPLIED PRODUCTION_CHANGED=LIVE_APP_AND_DESIRED_STATE PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=YES CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_PREAPPLY_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES PREAPPLY_BACKUP_REUSED=YES AST_IDENTICAL_NORMALIZATION=YES EXISTING_ROUTES_BASELINE=OK NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES OPENAPI_DOCS_EXPOSURE_CHANGED=NO CORE_DATABASE_STATE=1|1|9|7|1 INCIDENT_ENGINE=ACTIVE VM180_FAILED_UNITS=0 ROLLBACK_CHECKPOINT=VERIFIED OLD_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a NEW_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 READY_FOR_STAGE3_AUTHENTICATED_PRODUCTION_VERIFY=YES STAGE3_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-production-apply/20260713T032235Z -1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL -1146:/var/lib/homelab-health/special-vm-emergency-non-app-final-seal.txt:1:STATUS=OK TS=2026-07-14T01:15:48Z TYPE=special-vm-emergency-non-app-final-seal BAD=0 WARN=0 FAMILY_COVERAGE=11/11 MATRIX_STATUS=OK JOBS_TOTAL=40 JOBS_OK=36 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 COMPATIBILITY_OUTPUTS=31 REQUIRED_GREEN_JOBS=14 VM_FULL=OK VM_LOCAL_DUMPS=OK EDGE_VM_SOURCE=OK EDGE_VM_CLOUD=OK NEXTCLOUD_VM_SOURCE=homelab_vm_local_dumps_2cloud NEXTCLOUD_VM_CLOUD=OK EMERGENCY_PACK=OK SKLADCHIK_REPORTS_MONITOR=OK GOLDEN_STATE=OK ROUTER_RUNNING_CONFIG=OK MAIL_CLOUD_CRITICAL=OK MAIL_CLOUD_EDGE=OK MAIL_CLOUD_NEXTCLOUD=OK WRITE_MODE=legacy READ_MODE=dual MIGRATE_EXISTING=no SOURCE_BACKUP_RUN=NO CLOUD_OPERATIONS_RUN=NO RETENTION_RUN=NO CLOUD_PURGE_RUN=NO CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=b26f498b55de5d80bcbb4b9fb3a9bccded9f99b4 CURRENT_SPECIAL_VM_EMERGENCY_NON_APP_STAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/special-vm-emergency-non-app-final-seal/20260714T011514Z -1151:/var/lib/homelab-health/crypto-portfolio-ui-preapply-backup.txt:1:STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z -1157:/var/lib/homelab-health/legacy-monitoring-backups-migrated.txt:1:STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK -1159:/var/lib/homelab-health/cluster-admin-stage2-final-seal.txt:1:STATUS=OK TS=2026-07-13T01:45:29Z TYPE=cluster-admin-stage2-final-seal BAD=0 STAGE1_CLOSED=YES STAGE2_APPLY=OK PRODUCTION_CHANGED=BACKUP_HEALTH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_SCHEMA_CHANGED=NO_ADDITIONAL_CHANGE_DURING_SEAL INCIDENT_ENGINE_RESTARTED=NO MIGRATION_ID=002_canonical_foundation_sync FUNCTION=cluster_admin_sync_foundation COMMITTED_SYNC=YES MULTICYCLE_COLLECTOR_TEST=OK MULTICYCLE_NEW_RUNS=2 MULTICYCLE_GOOD_RUNS=2 ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK RETIRED_PLACEHOLDERS_ALLOWED=YES FOUNDATION_COUNTS=85|21|2|229|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0|0 ORPHANS=0 POSTAPPLY_DIRECT_SNAPSHOT=VERIFIED POSTAPPLY_RESTORE_TEST=OK POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T014757Z LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE2_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage2/stage2-final-seal-latest.txt REPORT_SHA256=32d4d37d82aa25194d789083d0e7ad2e9052ee38a72df5312b711226898cf1a6 BLOCKERS=/var/lib/homelab-cluster-admin-stage2/20260713T014529Z/blockers.txt -1169:/var/lib/homelab-health/cluster-admin-stage3-final-seal.txt:1:STATUS=OK TS=2026-07-13T03:46:30Z TYPE=cluster-admin-stage3-final-seal BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=HEALTH_REFRESH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO STAGE2_CLOSED=YES STAGE3_APPLY=OK AUTHENTICATED_PRODUCTION_VERIFY=OK AUTHENTICATED_PRODUCTION_TESTS=25 APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 NEW_GET_ROUTES=7 DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY OPENAPI_DOCS_EXPOSURE=PRESERVED_DISABLED POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z POSTAPPLY_APPBACKUP_RESTORE_CHECK=OK COLLECTOR_STABILITY=OK LATEST_COLLECTOR_RUN_ID=3114 CANONICAL_CONSISTENCY=0|0|0|0|0 OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE3_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage3/stage3-final-seal-latest.txt REPORT_SHA256=6f67545cd7f0a75b87146c1df74c61c3b88e142f9e53a96b083ec27209e6d265 -1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL -1172:/var/lib/homelab-health/backup-matrix-desired-state-persistence.txt:1:STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z -1175:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES -1184:/var/lib/homelab-health/cluster-admin-stage4d-adapter-candidates-in-isolation.txt:1:STATUS=OK TS=2026-07-14T06:14:58Z TYPE=cluster-admin-stage4d-adapter-candidates-in-isolation-final-seal BAD=0 STAGE4D_CLOSED=YES NEXT_STAGE=4E_RESTRICTED_PROBE_AGENT_DESIGN PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO APPLICATION_CHANGED=NO ENVELOPE_TOTAL=48 PROMETHEUS_ENVELOPES=5 BACKUP_MATRIX_ENVELOPES=40 HEALTH_ENVELOPES=3 LOCAL_REMOTE_OUTPUT_MATCH=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=e77fb37a0fa0033fe46558045e31dd92b6d1238a DESIRED_STATE_CLEAN=YES PVE01_FAILED_UNITS=0 REPORT=/var/lib/homelab-cluster-admin-stage4/20260714T061458Z/stage4d-adapter-candidates-in-isolation-seal.txt REPORT_SHA256=da6244c0a8a458b508fd58a0fec9cb1e6fa8da98a36f44d1368cbe947f4b5835 -1185:/var/lib/homelab-health/backup-matrix-cluster-desired-state-sync.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z -1187:/var/lib/homelab-health/cluster-admin-stage3-api-runtime-dry-run.txt:1:STATUS=OK TS=2026-07-13T02:53:34Z TYPE=cluster-admin-stage3-api-runtime-dry-run BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=TEMPORARY_ISOLATED_RUNTIME_AND_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO PRODUCTION_SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ISOLATED_BIND=127.0.0.1:18081 TEMPORARY_DATABASE_RESTORED=YES TEMPORARY_DATABASE_DROPPED=YES TEMPORARY_DATABASE_UNCHANGED=YES RUNTIME_TESTS=41 RUNTIME_FAILED_TESTS=0 EXISTING_ROUTES_BACKWARD_COMPATIBLE=YES NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES AUTHENTICATED_NEW_ROUTES=OK PAGINATION=OK FILTER_VALIDATION=OK ENTITY_DETAIL=OK ENTITY_DEPENDENCIES=OK SOURCES=OK POLICIES=OK COLLECTOR_RUNS=OK DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY FORBIDDEN_RESPONSE_FIELDS=ABSENT OPENAPI_DOCS_EXPOSURE_CHANGED=NO SECURITY_HEADERS=OK ISOLATED_RUNTIME_STOPPED=YES ISOLATED_PORT_CLOSED=YES LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK READY_FOR_STAGE3_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z RESULT=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z/runtime-result.json -1188:/var/lib/homelab-health/crypto-portfolio-remove-grt-failed-units-diagnostic.txt:1:STATUS=OK TS=2026-07-13T05:41:00Z TYPE=crypto-portfolio-remove-grt-failed-units-diagnostic BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=NO_DIAGNOSTIC_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO FAILED_STATE_RESET=NO CLOUD_WRITE_RUN=NO PVE01_FAILED_UNIT_COUNT=2 PVE01_FAILED_UNIT_NAMES=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service CRYPTO_PORTFOLIO_FAILED_UNIT_COUNT=0 OTHER_FAILED_UNIT_COUNT=2 CURRENT_CRYPTO_PRECHANGE_HEALTH=YES UNIT_DETAILS_CAPTURED=YES READY_FOR_TARGETED_FAILED_UNIT_DECISION=YES DIAGNOSTIC_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-failed-units-diagnostic/20260713T054100Z -1189:/var/lib/homelab-health/crypto-portfolio-remove-grt-production-apply.txt:1:STATUS=OK TS=2026-07-13T05:47:54Z TYPE=crypto-portfolio-remove-grt-production-apply BAD=0 COMMAND_PHYSICAL_LINES=1 APPLY_REVISION=V3 FIX_UID_READONLY=YES FIX_READINESS_ENDPOINT=YES FIX_ROLLBACK_DATABASE_METADATA=YES PRODUCTION_CHANGED=YES_REMOVE_GRT DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND PRODUCTION_CONTAINER_RECREATED=YES CLOUD_WRITE_RUN=NO ROLLBACK_ON_FAILURE=ARMED_NOT_NEEDED TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 PRECHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CHART_HEALTH_SCRIPT_SHA256=c16473f16390eeeff4d4eb701db604093564dd78d0be2d30b4cc8ffeeb22364a ACTIVE_COINS=HBAR,POL GRT_SOURCE_REFERENCES=0 GRT_DATABASE_ROWS=0|0|0 GRT_MARKET_ENTRY=ABSENT GRT_CACHE_ENTRIES=ABSENT DATABASE_MODE=0644 DATABASE_UID=10001 DATABASE_GID=10001 DATABASE_INTEGRITY=ok WORKER_STATUS=OK WORKER_COIN_COUNT=2 PUBLIC_CODES=200|303|401|200|200|401 CHART_HEALTH=OK CHART_SYMBOLS=HBAR,POL CHART_HEALTH_TIMER=ACTIVE VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_UNITS=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES REMOTE_STAGE_CLEANED=YES READY_FOR_GRT_POSTAPPLY_VERIFY_BACKUP_SYNC=YES APPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-production-apply-v3/20260713T054754Z -1193:/var/lib/homelab-health/crypto-portfolio-runtime-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES -1194:/var/lib/homelab-health/backup-matrix-cluster-preseal-convergence.txt:1:STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z -1202:/var/lib/homelab-health/node-loss-final-closure.txt:1:STATUS=OK TS=2026-07-08T16:35:56Z TYPE=node-loss-final-closure BAD=0 MODE=ALL_NODE_LOSS_DRILLS_SEALED PVE03_EDGE_RESTORE=OK PVE03_EDGE_FRESHNESS=OK PVE02_FORUM_RUNTIME=OK PVE01_NEXTCLOUD_DNS=OK PVE01_CORE_MONITORING_APPLEVEL=OK FULL_VM_GAP_VM170_VM171=ACKNOWLEDGED_APP_LEVEL_ONLY TEST_VMS_CLEANED=YES DNS_PAIR_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-node-loss/final-closure-latest.md -1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed -1332:/var/lib/homelab-health/codevipe-cloud-backup-reset.txt:1:STATUS=OK TS=2026-07-08T05:25:22Z TYPE=remote-failed-unit-codevipe-reset BAD=0 WARN=0 MODE=RESET_STALE_FAILED_STATE UNIT=codevipe-cloud-backup.service REASON=last_journal_success_and_no_timer ACTION=reset_failed_only -1426:● homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image -1499:2026-07-21T11:35:23Z BACKUP-DUAL-CLOUD-PREFLIGHT-059 c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479 1 FAIL backup-audit -1500:2026-07-21T12:28:26Z BACKUP-VM-DUMP-PIPELINE-073 cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b 1 FAIL backup-audit -1501:2026-07-21T12:42:53Z BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075 a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936 143 FAIL backup-audit -1502:2026-07-21T12:56:52Z BACKUP-DRAIN-LOCAL-ARCHIVES-079 531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32 75 FAIL backup-remediation -1503:2026-07-21T13:04:48Z BACKUP-FINAL-PROOF-082 9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601 127 FAIL backup-audit -1505:2026-07-21T13:14:27Z CLUSTER-BACKUP-POSTCLEAN-PROOF-086 601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da 127 FAIL backup-cleanup -1506:2026-07-21T13:15:21Z BACKUP-POLICY-CLOSEOUT-087 d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52 1 FAIL backup-cleanup -1507:2026-07-21T13:17:20Z BACKUP-CLEANUP-FINAL-CLOSEOUT-088 5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6 1 FAIL backup-cleanup -=== ACTIONABLE_P1 === -=== ACTIONABLE_P1 === -=== ACTIONABLE_P1 === -26:1:# HOMELAB ASSISTANT ERROR REGISTER -28:77:21. Ошибка: nested PHP php -r дал Parse error на forum-prod. -29:83:Команда 108 дала PHP Parse error на smtpHost и пустой proof-файл. -30:88:Команда 111 не изменила smtpSsl: OLD_SMTPSSL=false, SQL ERROR 1064, NEW_SMTPSSL=false. -40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL. -41:154:37. Coverage gap: pve03_staging missing from disk-space health coverage. -42:159:Факт: terminal output around command 222 showed paste artifact near "FAIL; fi". -45:171:Факт: first reconciliation wrote STATUS=ERROR because router-running-config-mail-cloud.txt is on pve01, not edge-vm. -51:194:- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200. -52:195:- If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed. -55:199:- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('. -60:227:- Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli]. -79:368:- Evidence: proof 542 showed all five zones missing and DNS record create probes failed. -83:388:- Impact: web routing is OK, but mail-related DNS may still contain stale provider data. -84:392:- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt. -85:397:- Context: SPF cleanup command 565 attempted to replace stale SPF records. -93:416:- Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line. -95:429:- Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth. -97:435:- Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory contains old codevipe-public/codevipe.sql files, not current five-forum archive. -120:621:- Certbot on edge rejected --dns-cloudflare-* arguments: dns-cloudflare plugin missing/not loaded. -129:657:- Факт: bash начал интерпретировать Python-код, появились syntax error и вторичный prompt >. -131:675:- When asked to fix one missing Uptime Kuma service, add only one monitor, then generate proposals separately. -134:681:- Dockge inactive items after migration can be stale compose folders, not stopped containers. -135:684:- Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions. -145:737:- The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults. -149:761:- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults. -153:774:- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects. -155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure. -171:839:- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`. -180:865:- Symptom: error-register candidate construction stopped with RC1 before applying the candidate. -183:876:- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis. -201:948:BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative. -203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (. -220:- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR. -235:- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring. -236:- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification. -245:- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first. -250:- Stale moved edge stack folders archived under /opt/dockge-stale-archive/20260702T230442Z on edge-vm; no runtime containers were deleted. -252:- Final target: remote not_visible count must be 0 and edge stale remaining matches must be 0. -258:- Stale moved stacks were archived, not deleted. -316:- Фактические поля collector_runs: finished_at и error_text; completed_at и error отсутствуют. -320:- Production осталась 0|0|OK, timer активен, failed units отсутствуют. -372: "missing": [ -459:STATUS=FAIL -475:JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000 -486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800 -495:JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000 -508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000 -510:JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000 -513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800 -676:2026-07-08T09:26:36.7403618890 390 /var/lib/homelab-health/dockge-stale-stack-classification-after-agents.txt -681:2026-07-08T09:26:36.7406026680 390 /var/lib/homelab-health/homepage-missing-card-proposals.txt -684:2026-07-08T09:26:36.7408259230 390 /var/lib/homelab-health/homepage-missing-card-proposals-final.txt -689:2026-07-08T09:26:36.7410552890 389 /var/lib/homelab-health/dockge-stale-archive-last.txt -1114:/var/lib/homelab-health/service-registry.txt:1:STATUS=OK TS=2026-07-21T13:15:17Z TYPE=service-registry SERVICES=12 MISSING=0 REGISTRY=/var/lib/homelab-registry/service-registry-latest.tsv -1122:/var/lib/homelab-health/nextcloud-restore-proof-offhost.txt:1:STATUS=OK TS=20260624T190330Z TYPE=nextcloud-restore-proof-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge/nextcloud-aio/nextcloud-aio-restore-proof/20260624T120344Z ITEMS=1 OK=1 ERROR=0 BYTES=2135684 FILES=7 SECRET_PRINTED=NO -1127:/var/lib/homelab-health/crypto-portfolio-remove-grt-recovery-audit.txt:1:STATUS=ERROR TS=2026-07-13T05:39:07Z TYPE=crypto-portfolio-remove-grt-recovery-audit BAD=1 COMMAND_PHYSICAL_LINES=1 ROOT_CAUSE=READINESS_ENDPOINT_MISMATCH RECOVERY_VERIFIED=NO TASK_COMPLETE=NO READY_FOR_GRT_APPLY_SCRIPT_FIX=NO AUDIT_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-recovery-audit-v2/20260713T053907Z -1133:/var/lib/homelab-health/final-readiness.txt:1:STATUS=WARN TS=2026-07-21T05:31:30Z TYPE=final-readiness BAD=5 PASS=48 FAIL=5 REPORT=/var/lib/homelab-final-readiness/latest.md -1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL -1152:/var/lib/homelab-health/router-hard-power.txt:1:STATUS=ERROR TS=2026-07-16T06:20:34Z MODE=cycle REMOTE=[PRIVATE_IP] RC=74 -1162:/var/lib/homelab-health/private-vpn-hosts.txt.bak-ok-wrapper-20260624T132705Z:16:STATUS=ERROR -1164:/var/lib/homelab-health/paperless-restore.txt:1:STATUS=STALE -1165:/var/lib/homelab-health/paperless-restore.txt:20:FRESHNESS=STALE -1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL -1174:/var/lib/homelab-health/ungated-health-backlog.txt:1:STATUS=OK TS=2026-07-07T07:45:01Z TYPE=ungated-health-backlog BAD=0 WARN=1 MODE=POST_PVE03_ROOT_CLEANUP_TRIAGE TOTAL=8 SUPERSEDED=4 STALE=2 POLICY_GAP_NEXT_STAGE=1 ACTIONABLE_CAPACITY=1 PVE03_ROOT_USAGE_PERCENT=31 PVE03_STAGING_USAGE_PERCENT=83 REPORT=/var/lib/homelab-ungated-health-backlog/latest.md -1190:/var/lib/homelab-health/mail-cloud-restore-drill.txt:1:STATUS=STALE -1191:/var/lib/homelab-health/mail-cloud-restore-drill.txt:20:FRESHNESS=STALE -1192:/var/lib/homelab-health/edge-restore-proofs-offhost.txt:1:STATUS=OK TS=20260624T190040Z TYPE=edge-restore-proofs-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge ITEMS=14 OK=14 ERROR=0 BYTES=226024287 SECRET_PRINTED=NO -1195:/var/lib/homelab-health/uptime-kuma-pending-health-monitors-fix.txt:8:NOTE=Paperless Restore may become red if its health file contains STATUS=ERROR; that is a real restore-health issue, not a Kuma pending issue. -1199:/var/lib/homelab-health/cluster-admin-stage0-convergence.txt:1:STATUS=WARN TS=2026-07-12T23:12:59Z TYPE=cluster-admin-stage0-targeted-convergence BAD=2 PROFILE_OK=NO CLUSTER_PASSPORT=FAIL FINAL_READINESS=FAIL OVERALL=FAIL DUTY_ADMIN=FAIL FAILED_UNITS=0 LIVE_MATCH_DESIRED_STATE=YES READY_FOR_FINAL_SEAL=NO REPORT_DIR=/var/lib/homelab-cluster-admin-stage0/readiness-profile-20260712T231259Z PRODUCTION_CHANGED=HEALTH_REGENERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO -1207:/var/lib/homelab-health/filebrowser-restore.txt:1:STATUS=STALE -1208:/var/lib/homelab-health/filebrowser-restore.txt:20:FRESHNESS=STALE -1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed -1494:2026-07-21T08:03:07Z GITEA-PUSH-AUTH-001 fcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94 1 FAIL gitea-runtime -1495:2026-07-21T08:13:32Z GITEA-TOKEN-CLI-HELP-003 f1d0f967eed6490ad822944a42108c82dc4b59c9fd2557d8c64624dcb6853cd4 1 FAIL gitea-runtime -1496:2026-07-21T08:14:52Z GITEA-CLI-CAPABILITIES-004 2cd6c4d28c8f646d8063105fca80b2f80d38c6a5db3a3eb76869de326b12d2d6 1 FAIL gitea-runtime -1497:2026-07-21T09:49:51Z RUNTIME-RECEIVER-STATE-027 0831002c78c26f35826cd7199995e335ceadcd39f08b9bac23882a0fbb59c8cc 127 FAIL gitea-runtime -1498:2026-07-21T10:26:18Z RUNTIME-RECEIVER-CLEANUP-036 65fbcfbce38a55723a92a8790429d6567389321f030f02cb4bf301cbe67e2531 1 FAIL gitea-runtime -1499:2026-07-21T11:35:23Z BACKUP-DUAL-CLOUD-PREFLIGHT-059 c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479 1 FAIL backup-audit -1500:2026-07-21T12:28:26Z BACKUP-VM-DUMP-PIPELINE-073 cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b 1 FAIL backup-audit -1501:2026-07-21T12:42:53Z BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075 a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936 143 FAIL backup-audit -1502:2026-07-21T12:56:52Z BACKUP-DRAIN-LOCAL-ARCHIVES-079 531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32 75 FAIL backup-remediation -1503:2026-07-21T13:04:48Z BACKUP-FINAL-PROOF-082 9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601 127 FAIL backup-audit -1504:2026-07-21T13:07:09Z CLUSTER-CLEANUP-INVENTORY-083 09d3f14ca0205936a910dc7af7991474030273a47d76fa033ea6a3185c083922 1 FAIL cluster-cleanup -1505:2026-07-21T13:14:27Z CLUSTER-BACKUP-POSTCLEAN-PROOF-086 601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da 127 FAIL backup-cleanup -1506:2026-07-21T13:15:21Z BACKUP-POLICY-CLOSEOUT-087 d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52 1 FAIL backup-cleanup -1507:2026-07-21T13:17:20Z BACKUP-CLEANUP-FINAL-CLOSEOUT-088 5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6 1 FAIL backup-cleanup -=== ACTIONABLE_STORAGE === -=== ACTIONABLE_STORAGE === -=== ACTIONABLE_STORAGE === -40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL. -454:local dir active 102626232 41916788 55450180 40.84% -=== OPEN_ERROR_REGISTER === -=== FIRST_PRIORITY_CANDIDATES === -38-BOOTSTRAP_SECTION_LINES=0 -39-=== ACTIONABLE_P0 === -40:86:398:- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained. -41:155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure. -42:203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (. -43:486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800 -44:508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000 -45:513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800 -46:546:● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -47:548:● homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -48:1102:/var/lib/homelab-health/backup-matrix-cluster-final-seal.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z -49:1104:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-production-closure.txt:1:STATUS=OK TS=2026-07-13T08:12:14Z TYPE=crypto-portfolio-ui-center-alert-currency-production-closure BAD=0 PRODUCTION_CHANGED=YES_UI_ONLY INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 POSTCHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION_ROLLBACK_COUNT=3 RETENTION_OPTIONAL_OTHER_CANDIDATES=1 RETENTION_REMOVED_ROLLBACK_COUNT=0 DANGLING_IMAGES_REMAINING=2 UNREFERENCED_DANGLING_IMAGES=0 DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE_CLEAN=YES PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES READY_FOR_UI_FINAL_SEAL=YES -50:1106:/var/lib/homelab-health/cluster-admin-stage3-postapply-backup.txt:1:STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z -51:1109:/var/lib/homelab-health/pve01-core-monitoring-app-restorecheck.txt:1:STATUS=OK TS=2026-07-08T15:57:27Z TYPE=pve01-core-monitoring-app-restorecheck BAD=0 MODE=APP_LEVEL_RESTORE_CHECKS_TWO_CLOUDS FULL_VM_RESTORE_AVAILABLE=NO TOTAL_APPS=30 OK_APPS=30 FAIL_LIST=none REPORT=/var/lib/homelab-node-loss/pve01-core-monitoring-app-level/app-restorecheck-latest.md -52:1118:/var/lib/homelab-health/hapusya-site-final-seal.txt:1:STATUS=OK TS=2026-07-13T12:33:08Z TYPE=hapusya-site-final-seal BAD=0 APP=hapusya-site URL=https://hapusya.ru/ PRESEAL_DESIRED_STATE_HEAD=69966eb83fc065141df3c46df0d6c0f282275046 ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb APPLY_MANIFEST_SHA256=f44992e3e0cc443b372779cb35d9e899d904a9960a9de03c14559886fdb1fc75 PRODUCTION_BROWSER_MANIFEST_SHA256=9f3cdf3063f8724d1d1c1ad619f47d6f98d434c3422a74903585146858a8dfac BROWSER_SMOKE=OK BROWSER_RUNS=5 BROWSER_ERRORS=0 DESKTOP_VIDEO=OK MOBILE_VIDEO=OK STATIC_FALLBACK=OK BACKUP_JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 TWO_CLOUDS=OK RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK DESIRED_STATE_SYNC=OK GENERATOR_SHA256=507bbbc2691f1f08cb1dd3660ed76cf13a8c36c23833d5fdec40cf6e07486cb8 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 POLICY_SHA256=4bf996ff451ae3ec56eb2fcd42842606cd97295b4d5141aedb135c956395e4b5 EVIDENCE_ROOT=/var/lib/homelab-change-sets/hapusya-site-final-seal/20260713T123308Z EVIDENCE_MANIFEST_SHA256=7eb993a17ac18a1f7c9ae0b52d745d3778833d15626ed34c82496f74916eb26a PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY DESIRED_STATE_CHANGED=YES_FINAL_SEAL_COMMIT_ONLY ALL_HAPUSYA_SITE_TAILS_CLOSED=YES TASK_COMPLETE=YES -53:1120:/var/lib/homelab-health/pve03-failed-unit-cleanup-closure.txt:1:STATUS=OK TS=2026-07-07T12:52:21Z TYPE=pve03-failed-unit-cleanup-closure BAD=0 WARN=0 MODE=READ_ONLY_CLOSURE LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CP_MATCH=YES RESTORE_CHECK=YES PVE03_FAILED_UNITS=0 REPORT=/var/lib/homelab-pve03-failed-unit-cleanup/closure-latest.md -54:1128:/var/lib/homelab-health/cluster-admin-backup-webpanel-final-seal.txt:1:STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[PRIVATE_IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md -55:1132:/var/lib/homelab-health/appbackupctl-routing.txt:1:STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z -56:1134:/var/lib/homelab-health/cluster-admin-stage3-api-apply.txt:1:STATUS=OK TS=2026-07-13T03:22:35Z TYPE=cluster-admin-stage3-api-apply BAD=0 COMMAND_PHYSICAL_LINES=1 RESULT=APPLIED PRODUCTION_CHANGED=LIVE_APP_AND_DESIRED_STATE PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=YES CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_PREAPPLY_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES PREAPPLY_BACKUP_REUSED=YES AST_IDENTICAL_NORMALIZATION=YES EXISTING_ROUTES_BASELINE=OK NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES OPENAPI_DOCS_EXPOSURE_CHANGED=NO CORE_DATABASE_STATE=1|1|9|7|1 INCIDENT_ENGINE=ACTIVE VM180_FAILED_UNITS=0 ROLLBACK_CHECKPOINT=VERIFIED OLD_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a NEW_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 READY_FOR_STAGE3_AUTHENTICATED_PRODUCTION_VERIFY=YES STAGE3_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-production-apply/20260713T032235Z -57:1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL -58:1146:/var/lib/homelab-health/special-vm-emergency-non-app-final-seal.txt:1:STATUS=OK TS=2026-07-14T01:15:48Z TYPE=special-vm-emergency-non-app-final-seal BAD=0 WARN=0 FAMILY_COVERAGE=11/11 MATRIX_STATUS=OK JOBS_TOTAL=40 JOBS_OK=36 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 COMPATIBILITY_OUTPUTS=31 REQUIRED_GREEN_JOBS=14 VM_FULL=OK VM_LOCAL_DUMPS=OK EDGE_VM_SOURCE=OK EDGE_VM_CLOUD=OK NEXTCLOUD_VM_SOURCE=homelab_vm_local_dumps_2cloud NEXTCLOUD_VM_CLOUD=OK EMERGENCY_PACK=OK SKLADCHIK_REPORTS_MONITOR=OK GOLDEN_STATE=OK ROUTER_RUNNING_CONFIG=OK MAIL_CLOUD_CRITICAL=OK MAIL_CLOUD_EDGE=OK MAIL_CLOUD_NEXTCLOUD=OK WRITE_MODE=legacy READ_MODE=dual MIGRATE_EXISTING=no SOURCE_BACKUP_RUN=NO CLOUD_OPERATIONS_RUN=NO RETENTION_RUN=NO CLOUD_PURGE_RUN=NO CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=b26f498b55de5d80bcbb4b9fb3a9bccded9f99b4 CURRENT_SPECIAL_VM_EMERGENCY_NON_APP_STAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/special-vm-emergency-non-app-final-seal/20260714T011514Z -59:1151:/var/lib/homelab-health/crypto-portfolio-ui-preapply-backup.txt:1:STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z -60:1157:/var/lib/homelab-health/legacy-monitoring-backups-migrated.txt:1:STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK -61:1159:/var/lib/homelab-health/cluster-admin-stage2-final-seal.txt:1:STATUS=OK TS=2026-07-13T01:45:29Z TYPE=cluster-admin-stage2-final-seal BAD=0 STAGE1_CLOSED=YES STAGE2_APPLY=OK PRODUCTION_CHANGED=BACKUP_HEALTH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_SCHEMA_CHANGED=NO_ADDITIONAL_CHANGE_DURING_SEAL INCIDENT_ENGINE_RESTARTED=NO MIGRATION_ID=002_canonical_foundation_sync FUNCTION=cluster_admin_sync_foundation COMMITTED_SYNC=YES MULTICYCLE_COLLECTOR_TEST=OK MULTICYCLE_NEW_RUNS=2 MULTICYCLE_GOOD_RUNS=2 ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK RETIRED_PLACEHOLDERS_ALLOWED=YES FOUNDATION_COUNTS=85|21|2|229|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0|0 ORPHANS=0 POSTAPPLY_DIRECT_SNAPSHOT=VERIFIED POSTAPPLY_RESTORE_TEST=OK POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T014757Z LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE2_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage2/stage2-final-seal-latest.txt REPORT_SHA256=32d4d37d82aa25194d789083d0e7ad2e9052ee38a72df5312b711226898cf1a6 BLOCKERS=/var/lib/homelab-cluster-admin-stage2/20260713T014529Z/blockers.txt -62:1169:/var/lib/homelab-health/cluster-admin-stage3-final-seal.txt:1:STATUS=OK TS=2026-07-13T03:46:30Z TYPE=cluster-admin-stage3-final-seal BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=HEALTH_REFRESH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO STAGE2_CLOSED=YES STAGE3_APPLY=OK AUTHENTICATED_PRODUCTION_VERIFY=OK AUTHENTICATED_PRODUCTION_TESTS=25 APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 NEW_GET_ROUTES=7 DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY OPENAPI_DOCS_EXPOSURE=PRESERVED_DISABLED POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z POSTAPPLY_APPBACKUP_RESTORE_CHECK=OK COLLECTOR_STABILITY=OK LATEST_COLLECTOR_RUN_ID=3114 CANONICAL_CONSISTENCY=0|0|0|0|0 OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE3_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage3/stage3-final-seal-latest.txt REPORT_SHA256=6f67545cd7f0a75b87146c1df74c61c3b88e142f9e53a96b083ec27209e6d265 -63:1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL -64:1172:/var/lib/homelab-health/backup-matrix-desired-state-persistence.txt:1:STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z -65:1175:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES -66:1184:/var/lib/homelab-health/cluster-admin-stage4d-adapter-candidates-in-isolation.txt:1:STATUS=OK TS=2026-07-14T06:14:58Z TYPE=cluster-admin-stage4d-adapter-candidates-in-isolation-final-seal BAD=0 STAGE4D_CLOSED=YES NEXT_STAGE=4E_RESTRICTED_PROBE_AGENT_DESIGN PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO APPLICATION_CHANGED=NO ENVELOPE_TOTAL=48 PROMETHEUS_ENVELOPES=5 BACKUP_MATRIX_ENVELOPES=40 HEALTH_ENVELOPES=3 LOCAL_REMOTE_OUTPUT_MATCH=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=e77fb37a0fa0033fe46558045e31dd92b6d1238a DESIRED_STATE_CLEAN=YES PVE01_FAILED_UNITS=0 REPORT=/var/lib/homelab-cluster-admin-stage4/20260714T061458Z/stage4d-adapter-candidates-in-isolation-seal.txt REPORT_SHA256=da6244c0a8a458b508fd58a0fec9cb1e6fa8da98a36f44d1368cbe947f4b5835 -67:1185:/var/lib/homelab-health/backup-matrix-cluster-desired-state-sync.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z -68:1187:/var/lib/homelab-health/cluster-admin-stage3-api-runtime-dry-run.txt:1:STATUS=OK TS=2026-07-13T02:53:34Z TYPE=cluster-admin-stage3-api-runtime-dry-run BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=TEMPORARY_ISOLATED_RUNTIME_AND_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO PRODUCTION_SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ISOLATED_BIND=127.0.0.1:18081 TEMPORARY_DATABASE_RESTORED=YES TEMPORARY_DATABASE_DROPPED=YES TEMPORARY_DATABASE_UNCHANGED=YES RUNTIME_TESTS=41 RUNTIME_FAILED_TESTS=0 EXISTING_ROUTES_BACKWARD_COMPATIBLE=YES NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES AUTHENTICATED_NEW_ROUTES=OK PAGINATION=OK FILTER_VALIDATION=OK ENTITY_DETAIL=OK ENTITY_DEPENDENCIES=OK SOURCES=OK POLICIES=OK COLLECTOR_RUNS=OK DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY FORBIDDEN_RESPONSE_FIELDS=ABSENT OPENAPI_DOCS_EXPOSURE_CHANGED=NO SECURITY_HEADERS=OK ISOLATED_RUNTIME_STOPPED=YES ISOLATED_PORT_CLOSED=YES LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK READY_FOR_STAGE3_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z RESULT=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z/runtime-result.json -69:1188:/var/lib/homelab-health/crypto-portfolio-remove-grt-failed-units-diagnostic.txt:1:STATUS=OK TS=2026-07-13T05:41:00Z TYPE=crypto-portfolio-remove-grt-failed-units-diagnostic BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=NO_DIAGNOSTIC_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO FAILED_STATE_RESET=NO CLOUD_WRITE_RUN=NO PVE01_FAILED_UNIT_COUNT=2 PVE01_FAILED_UNIT_NAMES=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service CRYPTO_PORTFOLIO_FAILED_UNIT_COUNT=0 OTHER_FAILED_UNIT_COUNT=2 CURRENT_CRYPTO_PRECHANGE_HEALTH=YES UNIT_DETAILS_CAPTURED=YES READY_FOR_TARGETED_FAILED_UNIT_DECISION=YES DIAGNOSTIC_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-failed-units-diagnostic/20260713T054100Z -70:1189:/var/lib/homelab-health/crypto-portfolio-remove-grt-production-apply.txt:1:STATUS=OK TS=2026-07-13T05:47:54Z TYPE=crypto-portfolio-remove-grt-production-apply BAD=0 COMMAND_PHYSICAL_LINES=1 APPLY_REVISION=V3 FIX_UID_READONLY=YES FIX_READINESS_ENDPOINT=YES FIX_ROLLBACK_DATABASE_METADATA=YES PRODUCTION_CHANGED=YES_REMOVE_GRT DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND PRODUCTION_CONTAINER_RECREATED=YES CLOUD_WRITE_RUN=NO ROLLBACK_ON_FAILURE=ARMED_NOT_NEEDED TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 PRECHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CHART_HEALTH_SCRIPT_SHA256=c16473f16390eeeff4d4eb701db604093564dd78d0be2d30b4cc8ffeeb22364a ACTIVE_COINS=HBAR,POL GRT_SOURCE_REFERENCES=0 GRT_DATABASE_ROWS=0|0|0 GRT_MARKET_ENTRY=ABSENT GRT_CACHE_ENTRIES=ABSENT DATABASE_MODE=0644 DATABASE_UID=10001 DATABASE_GID=10001 DATABASE_INTEGRITY=ok WORKER_STATUS=OK WORKER_COIN_COUNT=2 PUBLIC_CODES=200|303|401|200|200|401 CHART_HEALTH=OK CHART_SYMBOLS=HBAR,POL CHART_HEALTH_TIMER=ACTIVE VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_UNITS=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES REMOTE_STAGE_CLEANED=YES READY_FOR_GRT_POSTAPPLY_VERIFY_BACKUP_SYNC=YES APPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-production-apply-v3/20260713T054754Z -71:1193:/var/lib/homelab-health/crypto-portfolio-runtime-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES -72:1194:/var/lib/homelab-health/backup-matrix-cluster-preseal-convergence.txt:1:STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z -73:1202:/var/lib/homelab-health/node-loss-final-closure.txt:1:STATUS=OK TS=2026-07-08T16:35:56Z TYPE=node-loss-final-closure BAD=0 MODE=ALL_NODE_LOSS_DRILLS_SEALED PVE03_EDGE_RESTORE=OK PVE03_EDGE_FRESHNESS=OK PVE02_FORUM_RUNTIME=OK PVE01_NEXTCLOUD_DNS=OK PVE01_CORE_MONITORING_APPLEVEL=OK FULL_VM_GAP_VM170_VM171=ACKNOWLEDGED_APP_LEVEL_ONLY TEST_VMS_CLEANED=YES DNS_PAIR_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-node-loss/final-closure-latest.md -74:1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed -75:1332:/var/lib/homelab-health/codevipe-cloud-backup-reset.txt:1:STATUS=OK TS=2026-07-08T05:25:22Z TYPE=remote-failed-unit-codevipe-reset BAD=0 WARN=0 MODE=RESET_STALE_FAILED_STATE UNIT=codevipe-cloud-backup.service REASON=last_journal_success_and_no_timer ACTION=reset_failed_only -76-1426:● homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image -77:1499:2026-07-21T11:35:23Z BACKUP-DUAL-CLOUD-PREFLIGHT-059 c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479 1 FAIL backup-audit -78:1500:2026-07-21T12:28:26Z BACKUP-VM-DUMP-PIPELINE-073 cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b 1 FAIL backup-audit -79:1501:2026-07-21T12:42:53Z BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075 a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936 143 FAIL backup-audit -80:1502:2026-07-21T12:56:52Z BACKUP-DRAIN-LOCAL-ARCHIVES-079 531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32 75 FAIL backup-remediation -81:1503:2026-07-21T13:04:48Z BACKUP-FINAL-PROOF-082 9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601 127 FAIL backup-audit -82:1505:2026-07-21T13:14:27Z CLUSTER-BACKUP-POSTCLEAN-PROOF-086 601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da 127 FAIL backup-cleanup -83:1506:2026-07-21T13:15:21Z BACKUP-POLICY-CLOSEOUT-087 d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52 1 FAIL backup-cleanup -84:1507:2026-07-21T13:17:20Z BACKUP-CLEANUP-FINAL-CLOSEOUT-088 5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6 1 FAIL backup-cleanup -85-=== ACTIONABLE_P1 === -86-26:1:# HOMELAB ASSISTANT ERROR REGISTER -87-28:77:21. Ошибка: nested PHP php -r дал Parse error на forum-prod. -88-29:83:Команда 108 дала PHP Parse error на smtpHost и пустой proof-файл. --- -111-149:761:- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults. -112-153:774:- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects. -113:155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure. -114-171:839:- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`. -115-180:865:- Symptom: error-register candidate construction stopped with RC1 before applying the candidate. -116-183:876:- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis. -117-201:948:BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative. -118:203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (. -119-220:- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR. -120-235:- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring. -121-236:- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification. -122-245:- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first. --- -129-459:STATUS=FAIL -130-475:JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000 -131:486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800 -132-495:JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000 -133:508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000 -134-510:JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000 -135:513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800 -136-676:2026-07-08T09:26:36.7403618890 390 /var/lib/homelab-health/dockge-stale-stack-classification-after-agents.txt -137-681:2026-07-08T09:26:36.7406026680 390 /var/lib/homelab-health/homepage-missing-card-proposals.txt -138-684:2026-07-08T09:26:36.7408259230 390 /var/lib/homelab-health/homepage-missing-card-proposals-final.txt -139-689:2026-07-08T09:26:36.7410552890 389 /var/lib/homelab-health/dockge-stale-archive-last.txt --- -142-1127:/var/lib/homelab-health/crypto-portfolio-remove-grt-recovery-audit.txt:1:STATUS=ERROR TS=2026-07-13T05:39:07Z TYPE=crypto-portfolio-remove-grt-recovery-audit BAD=1 COMMAND_PHYSICAL_LINES=1 ROOT_CAUSE=READINESS_ENDPOINT_MISMATCH RECOVERY_VERIFIED=NO TASK_COMPLETE=NO READY_FOR_GRT_APPLY_SCRIPT_FIX=NO AUDIT_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-recovery-audit-v2/20260713T053907Z -143-1133:/var/lib/homelab-health/final-readiness.txt:1:STATUS=WARN TS=2026-07-21T05:31:30Z TYPE=final-readiness BAD=5 PASS=48 FAIL=5 REPORT=/var/lib/homelab-final-readiness/latest.md -144:1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL -145-1152:/var/lib/homelab-health/router-hard-power.txt:1:STATUS=ERROR TS=2026-07-16T06:20:34Z MODE=cycle REMOTE=[PRIVATE_IP] RC=74 -146-1162:/var/lib/homelab-health/private-vpn-hosts.txt.bak-ok-wrapper-20260624T132705Z:16:STATUS=ERROR -147-1164:/var/lib/homelab-health/paperless-restore.txt:1:STATUS=STALE -148-1165:/var/lib/homelab-health/paperless-restore.txt:20:FRESHNESS=STALE -149:1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL -150-1174:/var/lib/homelab-health/ungated-health-backlog.txt:1:STATUS=OK TS=2026-07-07T07:45:01Z TYPE=ungated-health-backlog BAD=0 WARN=1 MODE=POST_PVE03_ROOT_CLEANUP_TRIAGE TOTAL=8 SUPERSEDED=4 STALE=2 POLICY_GAP_NEXT_STAGE=1 ACTIONABLE_CAPACITY=1 PVE03_ROOT_USAGE_PERCENT=31 PVE03_STAGING_USAGE_PERCENT=83 REPORT=/var/lib/homelab-ungated-health-backlog/latest.md -151-1190:/var/lib/homelab-health/mail-cloud-restore-drill.txt:1:STATUS=STALE -152-1191:/var/lib/homelab-health/mail-cloud-restore-drill.txt:20:FRESHNESS=STALE -153-1192:/var/lib/homelab-health/edge-restore-proofs-offhost.txt:1:STATUS=OK TS=20260624T190040Z TYPE=edge-restore-proofs-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge ITEMS=14 OK=14 ERROR=0 BYTES=226024287 SECRET_PRINTED=NO --- -156-1207:/var/lib/homelab-health/filebrowser-restore.txt:1:STATUS=STALE -157-1208:/var/lib/homelab-health/filebrowser-restore.txt:20:FRESHNESS=STALE -158:1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed -159-1494:2026-07-21T08:03:07Z GITEA-PUSH-AUTH-001 fcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94 1 FAIL gitea-runtime -160-1495:2026-07-21T08:13:32Z GITEA-TOKEN-CLI-HELP-003 f1d0f967eed6490ad822944a42108c82dc4b59c9fd2557d8c64624dcb6853cd4 1 FAIL gitea-runtime -161-1496:2026-07-21T08:14:52Z GITEA-CLI-CAPABILITIES-004 2cd6c4d28c8f646d8063105fca80b2f80d38c6a5db3a3eb76869de326b12d2d6 1 FAIL gitea-runtime -162-1497:2026-07-21T09:49:51Z RUNTIME-RECEIVER-STATE-027 0831002c78c26f35826cd7199995e335ceadcd39f08b9bac23882a0fbb59c8cc 127 FAIL gitea-runtime -163-1498:2026-07-21T10:26:18Z RUNTIME-RECEIVER-CLEANUP-036 65fbcfbce38a55723a92a8790429d6567389321f030f02cb4bf301cbe67e2531 1 FAIL gitea-runtime -164:1499:2026-07-21T11:35:23Z BACKUP-DUAL-CLOUD-PREFLIGHT-059 c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479 1 FAIL backup-audit -165:1500:2026-07-21T12:28:26Z BACKUP-VM-DUMP-PIPELINE-073 cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b 1 FAIL backup-audit -166:1501:2026-07-21T12:42:53Z BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075 a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936 143 FAIL backup-audit -167:1502:2026-07-21T12:56:52Z BACKUP-DRAIN-LOCAL-ARCHIVES-079 531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32 75 FAIL backup-remediation -168:1503:2026-07-21T13:04:48Z BACKUP-FINAL-PROOF-082 9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601 127 FAIL backup-audit -169-1504:2026-07-21T13:07:09Z CLUSTER-CLEANUP-INVENTORY-083 09d3f14ca0205936a910dc7af7991474030273a47d76fa033ea6a3185c083922 1 FAIL cluster-cleanup -170:1505:2026-07-21T13:14:27Z CLUSTER-BACKUP-POSTCLEAN-PROOF-086 601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da 127 FAIL backup-cleanup -171:1506:2026-07-21T13:15:21Z BACKUP-POLICY-CLOSEOUT-087 d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52 1 FAIL backup-cleanup -172:1507:2026-07-21T13:17:20Z BACKUP-CLEANUP-FINAL-CLOSEOUT-088 5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6 1 FAIL backup-cleanup -173-=== ACTIONABLE_STORAGE === -174-40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL. -175-454:local dir active 102626232 41916788 55450180 40.84% -176-=== OPEN_ERROR_REGISTER === -=== OPEN_ERROR_REGISTER === -=== OPEN_ERROR_REGISTER === -NONE -=== RECENT_FAILED_COMMANDS_NORMALIZED === -CRITICAL_FIRST_ACTION=COMPLETE -=== CLUSTER_QUORUM === -Cluster information -------------------- -Name: homelab -Config Version: 3 -Transport: knet -Secure auth: on - -Quorum information ------------------- -Date: Tue Jul 21 17:39:59 2026 -Quorum provider: corosync_votequorum -Nodes: 3 -Node ID: 0x00000001 -Ring ID: 1.9d -Quorate: Yes - -Votequorum information ----------------------- -Expected votes: 3 -Highest expected: 3 -Total votes: 3 -Quorum: 2 -Flags: Quorate - -Membership information ----------------------- - Nodeid Votes Name -0x00000001 1 [PRIVATE_IP] (local) -0x00000002 1 [PRIVATE_IP] -0x00000003 1 [PRIVATE_IP] - -Membership information ----------------------- - Nodeid Votes Name - 1 1 pve01 (local) - 2 1 pve03 - 3 1 pve02 -=== FAILED_UNITS === - UNIT LOAD ACTIVE SUB DESCRIPTION -● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -● homelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01 -● homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 - -Legend: LOAD → Reflects whether the unit definition was properly loaded. - ACTIVE → The high-level unit activation state, i.e. generalization of SUB. - SUB → The low-level unit activation state, values depend on unit type. - -3 loaded units listed. -To show all installed unit files use 'systemctl list-unit-files'. ---- UNIT=codevipe-cloud-backup.service --- -Result=success -NRestarts=0 -ExecMainCode=0 -ExecMainStatus=0 -Id=codevipe-cloud-backup.service -LoadState=not-found -ActiveState=inactive -SubState=dead -FragmentPath= --- No entries -- ---- UNIT=homelab-backup-matrix-refresh.service --- -Result=exit-code -NRestarts=0 -ExecMainCode=1 -ExecMainStatus=1 -Id=homelab-backup-matrix-refresh.service -LoadState=loaded -ActiveState=failed -SubState=failed -FragmentPath=/etc/systemd/system/homelab-backup-matrix-refresh.service -Jul 21 05:46:46 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_STATUS=FAIL -Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_TOTAL=41 -Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_OK=30 -Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_BAD=6 -Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_UNMAPPED=0 -Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_DISABLED=5 -Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -Jul 21 05:46:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 05:46:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 05:46:47 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Jul 21 06:06:46 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_STATUS=FAIL -Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_TOTAL=41 -Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_OK=30 -Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_BAD=6 -Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_UNMAPPED=0 -Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_DISABLED=5 -Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -Jul 21 06:06:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 06:06:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 06:06:47 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Jul 21 06:45:16 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_STATUS=FAIL -Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_TOTAL=41 -Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_OK=30 -Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_BAD=6 -Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_UNMAPPED=0 -Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_DISABLED=5 -Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -Jul 21 06:45:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 06:45:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 06:45:17 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Jul 21 07:05:43 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_STATUS=FAIL -Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_TOTAL=41 -Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_OK=30 -Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_BAD=6 -Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_UNMAPPED=0 -Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_DISABLED=5 -Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -Jul 21 07:05:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 07:05:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 07:05:44 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Jul 21 07:36:59 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_STATUS=FAIL -Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_TOTAL=41 -Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_OK=30 -Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_BAD=6 -Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_UNMAPPED=0 -Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_DISABLED=5 -Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -Jul 21 07:37:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 07:37:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 07:37:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Jul 21 08:07:26 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_STATUS=FAIL -Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_TOTAL=41 -Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_OK=30 -Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_BAD=6 -Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_UNMAPPED=0 -Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_DISABLED=5 -Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -Jul 21 08:07:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 08:07:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 08:07:27 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Jul 21 08:30:13 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_STATUS=FAIL -Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_TOTAL=41 -Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_OK=30 -Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_BAD=6 -Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_UNMAPPED=0 -Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_DISABLED=5 -Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -Jul 21 08:30:14 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 08:30:14 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 08:30:14 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Jul 21 08:41:14 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_STATUS=FAIL -Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_TOTAL=41 -Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_OK=30 -Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_BAD=6 -Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_UNMAPPED=0 -Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_DISABLED=5 -Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -Jul 21 08:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 08:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 08:41:15 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Jul 21 08:50:15 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_STATUS=FAIL -Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_TOTAL=41 -Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_OK=30 -Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_BAD=6 -Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_UNMAPPED=0 -Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_DISABLED=5 -Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -Jul 21 08:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 08:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 08:50:16 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -Jul 21 10:02:41 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_STATUS=FAIL -Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_TOTAL=41 -Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_OK=30 -Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_BAD=6 -Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_UNMAPPED=0 -Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_DISABLED=5 -Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -Jul 21 10:02:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 10:02:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 10:02:42 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. ---- UNIT=homelab-cluster-admin-incident-engine-backup.service --- -Result=success -NRestarts=0 -ExecMainCode=1 -ExecMainStatus=0 -Id=homelab-cluster-admin-incident-engine-backup.service -LoadState=loaded -ActiveState=inactive -SubState=dead -FragmentPath=/etc/systemd/system/homelab-cluster-admin-incident-engine-backup.service -Jul 13 07:33:14 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 13 07:33:24 pve01 homelab-cluster-admin-incident-engine-backup-run[2386922]: FLAT_CHUNKED_COPY_OK SIZE=1818333 PARTS=1 -Jul 13 07:33:28 pve01 homelab-cluster-admin-incident-engine-backup-run[2387061]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T043314Z -Jul 13 07:33:37 pve01 homelab-cluster-admin-incident-engine-backup-run[2387287]: FLAT_CHUNKED_COPY_OK SIZE=1818333 PARTS=1 -Jul 13 07:33:38 pve01 homelab-cluster-admin-incident-engine-backup-run[2387448]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T043314Z -Jul 13 07:33:39 pve01 homelab-cluster-admin-incident-engine-backup-run[2386743]: [SUCCESS] short flat application uploaded and verified -Jul 13 07:33:39 pve01 homelab-cluster-admin-incident-engine-backup-run[2386712]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 13 07:33:39 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 13 07:33:39 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 13 07:33:39 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.357s CPU time, 50.3M memory peak. -Jul 14 04:46:45 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 14 04:46:58 pve01 homelab-cluster-admin-incident-engine-backup-run[2275200]: FLAT_CHUNKED_COPY_OK SIZE=2493105 PARTS=1 -Jul 14 04:46:58 pve01 homelab-cluster-admin-incident-engine-backup-run[2275366]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T014645Z -Jul 14 04:47:12 pve01 homelab-cluster-admin-incident-engine-backup-run[2275633]: FLAT_CHUNKED_COPY_OK SIZE=2493105 PARTS=1 -Jul 14 04:47:12 pve01 homelab-cluster-admin-incident-engine-backup-run[2275815]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T014645Z -Jul 14 04:47:18 pve01 homelab-cluster-admin-incident-engine-backup-run[2275029]: [SUCCESS] short flat application uploaded and verified -Jul 14 04:47:18 pve01 homelab-cluster-admin-incident-engine-backup-run[2274993]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 14 04:47:18 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 14 04:47:18 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 14 04:47:18 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.491s CPU time, 54.3M memory peak. -Jul 14 04:54:18 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 14 04:54:33 pve01 homelab-cluster-admin-incident-engine-backup-run[2297844]: FLAT_CHUNKED_COPY_OK SIZE=2505726 PARTS=1 -Jul 14 04:54:33 pve01 homelab-cluster-admin-incident-engine-backup-run[2298021]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T015418Z -Jul 14 04:54:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2298273]: FLAT_CHUNKED_COPY_OK SIZE=2505726 PARTS=1 -Jul 14 04:54:50 pve01 homelab-cluster-admin-incident-engine-backup-run[2298435]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T015418Z -Jul 14 04:54:51 pve01 homelab-cluster-admin-incident-engine-backup-run[2297589]: [SUCCESS] short flat application uploaded and verified -Jul 14 04:54:51 pve01 homelab-cluster-admin-incident-engine-backup-run[2297401]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 14 04:54:51 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 14 04:54:51 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 14 04:54:51 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.453s CPU time, 55M memory peak. -Jul 14 07:32:36 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 14 07:32:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2862512]: FLAT_CHUNKED_COPY_OK SIZE=2584458 PARTS=1 -Jul 14 07:32:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2862665]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T043236Z -Jul 14 07:33:02 pve01 homelab-cluster-admin-incident-engine-backup-run[2862911]: FLAT_CHUNKED_COPY_OK SIZE=2584458 PARTS=1 -Jul 14 07:33:05 pve01 homelab-cluster-admin-incident-engine-backup-run[2863090]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T043236Z -Jul 14 07:33:06 pve01 homelab-cluster-admin-incident-engine-backup-run[2862304]: [SUCCESS] short flat application uploaded and verified -Jul 14 07:33:06 pve01 homelab-cluster-admin-incident-engine-backup-run[2862165]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 14 07:33:06 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 14 07:33:06 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 14 07:33:06 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.436s CPU time, 54.9M memory peak. -Jul 14 09:43:27 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 14 09:43:39 pve01 homelab-cluster-admin-incident-engine-backup-run[3315449]: FLAT_CHUNKED_COPY_OK SIZE=2651545 PARTS=1 -Jul 14 09:43:42 pve01 homelab-cluster-admin-incident-engine-backup-run[3315701]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T064327Z -Jul 14 09:43:51 pve01 homelab-cluster-admin-incident-engine-backup-run[3315929]: FLAT_CHUNKED_COPY_OK SIZE=2651545 PARTS=1 -Jul 14 09:43:52 pve01 homelab-cluster-admin-incident-engine-backup-run[3316098]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T064327Z -Jul 14 09:43:55 pve01 homelab-cluster-admin-incident-engine-backup-run[3315291]: [SUCCESS] short flat application uploaded and verified -Jul 14 09:43:55 pve01 homelab-cluster-admin-incident-engine-backup-run[3315256]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 14 09:43:55 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 14 09:43:55 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 14 09:43:55 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.483s CPU time, 55.4M memory peak. -Jul 15 07:35:00 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 15 07:35:11 pve01 homelab-cluster-admin-incident-engine-backup-run[3512923]: FLAT_CHUNKED_COPY_OK SIZE=3303993 PARTS=1 -Jul 15 07:35:11 pve01 homelab-cluster-admin-incident-engine-backup-run[3513093]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260715T043500Z -Jul 15 07:35:30 pve01 homelab-cluster-admin-incident-engine-backup-run[3513650]: FLAT_CHUNKED_COPY_OK SIZE=3303993 PARTS=1 -Jul 15 07:35:31 pve01 homelab-cluster-admin-incident-engine-backup-run[3513799]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260715T043500Z -Jul 15 07:35:34 pve01 homelab-cluster-admin-incident-engine-backup-run[3512747]: [SUCCESS] short flat application uploaded and verified -Jul 15 07:35:34 pve01 homelab-cluster-admin-incident-engine-backup-run[3512710]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 15 07:35:34 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 15 07:35:34 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 15 07:35:34 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.396s CPU time, 59.3M memory peak. -Jul 16 07:32:14 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 16 07:32:23 pve01 homelab-cluster-admin-incident-engine-backup-run[95619]: FLAT_CHUNKED_COPY_OK SIZE=4089435 PARTS=1 -Jul 16 07:32:24 pve01 homelab-cluster-admin-incident-engine-backup-run[95775]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260716T043214Z -Jul 16 07:32:34 pve01 homelab-cluster-admin-incident-engine-backup-run[95997]: FLAT_CHUNKED_COPY_OK SIZE=4089435 PARTS=1 -Jul 16 07:32:35 pve01 homelab-cluster-admin-incident-engine-backup-run[96153]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260716T043214Z -Jul 16 07:32:36 pve01 homelab-cluster-admin-incident-engine-backup-run[95482]: [SUCCESS] short flat application uploaded and verified -Jul 16 07:32:36 pve01 homelab-cluster-admin-incident-engine-backup-run[95370]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 16 07:32:36 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 16 07:32:36 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 16 07:32:36 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.646s CPU time, 66.1M memory peak. --- Boot 9fe27497bd1642a386cb88c33185f42f -- -Jul 17 07:30:49 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 17 07:31:02 pve01 homelab-cluster-admin-incident-engine-backup-run[4051907]: FLAT_CHUNKED_COPY_OK SIZE=4782437 PARTS=1 -Jul 17 07:31:03 pve01 homelab-cluster-admin-incident-engine-backup-run[4052105]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260717T043049Z -Jul 17 07:31:17 pve01 homelab-cluster-admin-incident-engine-backup-run[4052291]: FLAT_CHUNKED_COPY_OK SIZE=4782437 PARTS=1 -Jul 17 07:31:20 pve01 homelab-cluster-admin-incident-engine-backup-run[4052752]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260717T043049Z -Jul 17 07:31:24 pve01 homelab-cluster-admin-incident-engine-backup-run[4051765]: [SUCCESS] short flat application uploaded and verified -Jul 17 07:31:24 pve01 homelab-cluster-admin-incident-engine-backup-run[4051685]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 17 07:31:24 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 17 07:31:24 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 17 07:31:24 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.642s CPU time, 68.6M memory peak. -Jul 18 07:31:41 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 18 07:31:54 pve01 homelab-cluster-admin-incident-engine-backup-run[685264]: FLAT_CHUNKED_COPY_OK SIZE=5630582 PARTS=1 -Jul 18 07:31:55 pve01 homelab-cluster-admin-incident-engine-backup-run[685475]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260718T043141Z -Jul 18 07:32:06 pve01 homelab-cluster-admin-incident-engine-backup-run[685660]: FLAT_CHUNKED_COPY_OK SIZE=5630582 PARTS=1 -Jul 18 07:32:07 pve01 homelab-cluster-admin-incident-engine-backup-run[685846]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260718T043141Z -Jul 18 07:32:08 pve01 homelab-cluster-admin-incident-engine-backup-run[685098]: [SUCCESS] short flat application uploaded and verified -Jul 18 07:32:08 pve01 homelab-cluster-admin-incident-engine-backup-run[684962]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 18 07:32:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 18 07:32:08 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 18 07:32:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.695s CPU time, 73.5M memory peak. -Jul 19 07:26:05 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 19 07:26:14 pve01 homelab-cluster-admin-incident-engine-backup-run[1415764]: FLAT_CHUNKED_COPY_OK SIZE=6386016 PARTS=1 -Jul 19 07:26:14 pve01 homelab-cluster-admin-incident-engine-backup-run[1415903]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260719T042605Z -Jul 19 07:26:31 pve01 homelab-cluster-admin-incident-engine-backup-run[1416145]: FLAT_CHUNKED_COPY_OK SIZE=6386016 PARTS=1 -Jul 19 07:26:31 pve01 homelab-cluster-admin-incident-engine-backup-run[1416534]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260719T042605Z -Jul 19 07:26:32 pve01 homelab-cluster-admin-incident-engine-backup-run[1415638]: [SUCCESS] short flat application uploaded and verified -Jul 19 07:26:32 pve01 homelab-cluster-admin-incident-engine-backup-run[1415567]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 19 07:26:32 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 19 07:26:32 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 19 07:26:32 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.704s CPU time, 78.3M memory peak. -Jul 20 07:37:35 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 20 07:37:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2212986]: FLAT_CHUNKED_COPY_OK SIZE=7183272 PARTS=1 -Jul 20 07:37:52 pve01 homelab-cluster-admin-incident-engine-backup-run[2213181]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260720T043735Z -Jul 20 07:38:04 pve01 homelab-cluster-admin-incident-engine-backup-run[2213436]: FLAT_CHUNKED_COPY_OK SIZE=7183272 PARTS=1 -Jul 20 07:38:07 pve01 homelab-cluster-admin-incident-engine-backup-run[2213580]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260720T043735Z -Jul 20 07:38:08 pve01 homelab-cluster-admin-incident-engine-backup-run[2212839]: [SUCCESS] short flat application uploaded and verified -Jul 20 07:38:08 pve01 homelab-cluster-admin-incident-engine-backup-run[2212719]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 20 07:38:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 20 07:38:08 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 20 07:38:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.798s CPU time, 83M memory peak. -Jul 21 07:36:14 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup... -Jul 21 07:36:26 pve01 homelab-cluster-admin-incident-engine-backup-run[3010900]: FLAT_CHUNKED_COPY_OK SIZE=8130454 PARTS=1 -Jul 21 07:36:27 pve01 homelab-cluster-admin-incident-engine-backup-run[3011053]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260721T043614Z -Jul 21 07:36:36 pve01 homelab-cluster-admin-incident-engine-backup-run[3011255]: FLAT_CHUNKED_COPY_OK SIZE=8130454 PARTS=1 -Jul 21 07:36:37 pve01 homelab-cluster-admin-incident-engine-backup-run[3011413]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260721T043614Z -Jul 21 07:36:40 pve01 homelab-cluster-admin-incident-engine-backup-run[3010739]: [SUCCESS] short flat application uploaded and verified -Jul 21 07:36:40 pve01 homelab-cluster-admin-incident-engine-backup-run[3010566]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK -Jul 21 07:36:40 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully. -Jul 21 07:36:40 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup. -Jul 21 07:36:40 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.760s CPU time, 88.3M memory peak. ---- UNIT=homelab-cluster-admin-pull.service --- -Result=exit-code -NRestarts=0 -ExecMainCode=1 -ExecMainStatus=1 -Id=homelab-cluster-admin-pull.service -LoadState=loaded -ActiveState=failed -SubState=failed -FragmentPath=/etc/systemd/system/homelab-cluster-admin-pull.service -Jul 21 01:06:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 01:11:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 01:11:36 pve01 homelab-cluster-admin-pull[1709417]: STATUS=WARN TS=2026-07-20T22:11:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 01:11:36 pve01 homelab-cluster-admin-pull[1709418]: STATUS=WARN TS=2026-07-20T22:11:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 01:11:36 pve01 homelab-cluster-admin-pull[1709419]: STATUS=WARN TS=2026-07-20T22:11:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 01:11:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:11:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 01:11:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 01:16:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 01:16:36 pve01 homelab-cluster-admin-pull[1733408]: STATUS=WARN TS=2026-07-20T22:16:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 01:16:36 pve01 homelab-cluster-admin-pull[1733409]: STATUS=WARN TS=2026-07-20T22:16:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 01:16:36 pve01 homelab-cluster-admin-pull[1733410]: STATUS=WARN TS=2026-07-20T22:16:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 01:16:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:16:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 01:16:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 01:21:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 01:21:36 pve01 homelab-cluster-admin-pull[1749007]: STATUS=WARN TS=2026-07-20T22:21:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 01:21:36 pve01 homelab-cluster-admin-pull[1749008]: STATUS=WARN TS=2026-07-20T22:21:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 01:21:36 pve01 homelab-cluster-admin-pull[1749009]: STATUS=WARN TS=2026-07-20T22:21:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 01:21:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:21:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 01:21:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 01:26:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 01:26:36 pve01 homelab-cluster-admin-pull[1764598]: STATUS=WARN TS=2026-07-20T22:26:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 01:26:36 pve01 homelab-cluster-admin-pull[1764599]: STATUS=WARN TS=2026-07-20T22:26:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 01:26:36 pve01 homelab-cluster-admin-pull[1764600]: STATUS=WARN TS=2026-07-20T22:26:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 01:26:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:26:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 01:26:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 01:31:45 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 01:31:52 pve01 homelab-cluster-admin-pull[1788839]: STATUS=WARN TS=2026-07-20T22:31:45Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 01:31:52 pve01 homelab-cluster-admin-pull[1788840]: STATUS=WARN TS=2026-07-20T22:31:45Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 01:31:52 pve01 homelab-cluster-admin-pull[1788841]: STATUS=WARN TS=2026-07-20T22:31:45Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 01:31:52 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:31:52 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 01:31:52 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 01:37:07 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 01:37:07 pve01 homelab-cluster-admin-pull[1804555]: STATUS=WARN TS=2026-07-20T22:37:07Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 01:37:07 pve01 homelab-cluster-admin-pull[1804556]: STATUS=WARN TS=2026-07-20T22:37:07Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 01:37:07 pve01 homelab-cluster-admin-pull[1804557]: STATUS=WARN TS=2026-07-20T22:37:07Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 01:37:07 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:37:07 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 01:37:07 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 01:42:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 01:42:15 pve01 homelab-cluster-admin-pull[1820229]: STATUS=WARN TS=2026-07-20T22:42:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 01:42:15 pve01 homelab-cluster-admin-pull[1820230]: STATUS=WARN TS=2026-07-20T22:42:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 01:42:15 pve01 homelab-cluster-admin-pull[1820231]: STATUS=WARN TS=2026-07-20T22:42:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 01:42:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:42:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 01:42:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 01:47:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 01:47:15 pve01 homelab-cluster-admin-pull[1844123]: STATUS=WARN TS=2026-07-20T22:47:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 01:47:15 pve01 homelab-cluster-admin-pull[1844124]: STATUS=WARN TS=2026-07-20T22:47:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 01:47:15 pve01 homelab-cluster-admin-pull[1844125]: STATUS=WARN TS=2026-07-20T22:47:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 01:47:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:47:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 01:47:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 01:52:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 01:52:15 pve01 homelab-cluster-admin-pull[1859657]: STATUS=WARN TS=2026-07-20T22:52:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 01:52:15 pve01 homelab-cluster-admin-pull[1859658]: STATUS=WARN TS=2026-07-20T22:52:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 01:52:15 pve01 homelab-cluster-admin-pull[1859659]: STATUS=WARN TS=2026-07-20T22:52:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 01:52:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:52:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 01:52:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 01:57:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 01:57:15 pve01 homelab-cluster-admin-pull[1875102]: STATUS=WARN TS=2026-07-20T22:57:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 01:57:15 pve01 homelab-cluster-admin-pull[1875103]: STATUS=WARN TS=2026-07-20T22:57:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 01:57:15 pve01 homelab-cluster-admin-pull[1875104]: STATUS=WARN TS=2026-07-20T22:57:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 01:57:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:57:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 01:57:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 02:02:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 02:02:15 pve01 homelab-cluster-admin-pull[1899107]: STATUS=WARN TS=2026-07-20T23:02:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 02:02:15 pve01 homelab-cluster-admin-pull[1899108]: STATUS=WARN TS=2026-07-20T23:02:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 02:02:15 pve01 homelab-cluster-admin-pull[1899109]: STATUS=WARN TS=2026-07-20T23:02:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 02:02:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:02:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 02:02:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 02:07:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 02:07:15 pve01 homelab-cluster-admin-pull[1914657]: STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 02:07:15 pve01 homelab-cluster-admin-pull[1914658]: STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 02:07:15 pve01 homelab-cluster-admin-pull[1914659]: STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 02:07:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:07:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 02:07:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 02:12:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 02:12:15 pve01 homelab-cluster-admin-pull[1930715]: STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 02:12:15 pve01 homelab-cluster-admin-pull[1930716]: STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 02:12:15 pve01 homelab-cluster-admin-pull[1930717]: STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 02:12:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:12:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 02:12:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 02:14:04 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 02:14:11 pve01 homelab-cluster-admin-pull[1932800]: STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 02:14:11 pve01 homelab-cluster-admin-pull[1932801]: STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 02:14:11 pve01 homelab-cluster-admin-pull[1932802]: STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 02:14:11 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:14:11 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 02:14:11 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 02:20:13 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 02:20:20 pve01 homelab-cluster-admin-pull[1958372]: STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 02:20:20 pve01 homelab-cluster-admin-pull[1958373]: STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 02:20:20 pve01 homelab-cluster-admin-pull[1958374]: STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 02:20:20 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:20:20 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 02:20:20 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 02:35:05 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 02:35:12 pve01 homelab-cluster-admin-pull[2013630]: STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 02:35:12 pve01 homelab-cluster-admin-pull[2013631]: STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 02:35:12 pve01 homelab-cluster-admin-pull[2013632]: STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 02:35:12 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:35:12 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 02:35:12 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. -Jul 21 02:36:57 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01... -Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015088]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt -Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015089]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md -Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015090]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer -Jul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'. -Jul 21 02:37:04 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01. ---- UNIT=homelab-edge-vm-offhost-freshness.timer --- -Result=success -Id=homelab-edge-vm-offhost-freshness.timer -LoadState=loaded -ActiveState=active -SubState=waiting -FragmentPath=/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer -Jul 16 09:20:43 pve01 systemd[1]: homelab-edge-vm-offhost-freshness.timer: Deactivated successfully. -Jul 16 09:20:43 pve01 systemd[1]: Stopped homelab-edge-vm-offhost-freshness.timer - Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump. --- Boot 9fe27497bd1642a386cb88c33185f42f -- -Jul 16 11:38:48 pve01 systemd[1]: Started homelab-edge-vm-offhost-freshness.timer - Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump. ---- UNIT=homelab-immutable-offline-verify.service --- -Result=success -NRestarts=0 -ExecMainCode=0 -ExecMainStatus=0 -Id=homelab-immutable-offline-verify.service -LoadState=not-found -ActiveState=inactive -SubState=dead -FragmentPath= --- No entries -- ---- UNIT=homelab-vm160-remote-backup.service --- -Result=exit-code -NRestarts=0 -ExecMainCode=1 -ExecMainStatus=1 -Id=homelab-vm160-remote-backup.service -LoadState=loaded -ActiveState=failed -SubState=failed -FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service -Jul 18 01:02:41 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -Jul 18 01:15:43 pve01 homelab-vm160-remote-backup[3491061]: VM160_TARGET_FINALIZE_STATUS=OK -Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3530071]: VM160_TARGET_PROMOTE_STATUS=OK -Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: VM160_REMOTE_BACKUP_STATUS=OK -Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z -Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z/vzdump-qemu-160-20260717T220242Z.vma.zst -Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: SIZE_BYTES=84993482488 -Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: SHA256=a9562d8b60d7fd5e7ec73bc43d252cfea80ed12c05b32cca0a73d7bb88532411 -Jul 18 01:19:10 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully. -Jul 18 01:19:10 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Jul 18 01:19:10 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak. -Jul 18 01:38:18 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -Jul 18 01:51:18 pve01 homelab-vm160-remote-backup[3609200]: VM160_TARGET_FINALIZE_STATUS=OK -Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3648126]: VM160_TARGET_PROMOTE_STATUS=OK -Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: VM160_REMOTE_BACKUP_STATUS=OK -Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z -Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst -Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: SIZE_BYTES=84995216465 -Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72 -Jul 18 01:54:48 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully. -Jul 18 01:54:48 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Jul 18 01:54:48 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak. -Jul 18 04:16:25 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -Jul 18 04:16:27 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 18 04:16:27 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 18 04:16:27 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Jul 19 04:19:14 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -Jul 19 04:19:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 04:19:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 19 04:19:16 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Jul 20 04:14:14 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -Jul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 20 04:14:16 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Jul 20 19:28:57 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -Jul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 20 19:28:59 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Jul 21 02:14:11 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -Jul 21 02:14:13 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:14:13 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 21 02:14:13 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Jul 21 02:20:20 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -Jul 21 02:20:21 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:20:21 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 21 02:20:21 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -Jul 21 02:39:12 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03... -Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 21 02:39:14 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03. -=== PROXMOX_STORAGE === -Name Type Status Total (KiB) Used (KiB) Available (KiB) % -local dir active 102626232 41936392 55430576 40.86% -local-lvm lvmthin active 848797696 280188119 568609576 33.01% -Filesystem Type Size Used Avail Use% Mounted on -udev devtmpfs 15G 0 15G 0% /dev -tmpfs tmpfs 3.2G 3.8M 3.2G 1% /run -/dev/mapper/pve-root ext4 98G 40G 53G 44% / -tmpfs tmpfs 16G 66M 16G 1% /dev/shm -efivarfs efivarfs 192K 135K 53K 72% /sys/firmware/efi/efivars -tmpfs tmpfs 5.0M 0 5.0M 0% /run/lock -tmpfs tmpfs 1.0M 0 1.0M 0% /run/credentials/systemd-journald.service -tmpfs tmpfs 16G 48M 16G 1% /tmp -/dev/nvme0n1p2 vfat 1022M 9.1M 1013M 1% /boot/efi -/dev/mapper/pve-homelab--staging ext4 295G 148G 133G 53% /mnt/staging -/dev/fuse fuse 128M 1.4M 127M 2% /etc/pve -shm tmpfs 63M 0 63M 0% /var/lib/containers/storage/overlay-containers/fbb240ea5b547fa140d8edbc3b16daaf4b28a21c7ee424abedab068865a95d5b/userdata/shm -overlay overlay 98G 40G 53G 44% /var/lib/containers/storage/overlay/4f68ccfa2802b35db2a4c6b396e477f756cb85aa42fa818dd111745ec503616b/merged -tmpfs tmpfs 1.0M 0 1.0M 0% /run/credentials/getty@tty1.service -tmpfs tmpfs 3.2G 4.0K 3.2G 1% /run/user/0 -Filesystem Inodes IUsed IFree IUse% Mounted on -udev 3.6M 733 3.6M 1% /dev -tmpfs 3.9M 1.4K 3.9M 1% /run -/dev/mapper/pve-root 6.3M 128K 6.2M 2% / -tmpfs 3.9M 137 3.9M 1% /dev/shm -efivarfs 0 0 0 - /sys/firmware/efi/efivars -tmpfs 3.9M 32 3.9M 1% /run/lock -tmpfs 1.0K 1 1023 1% /run/credentials/systemd-journald.service -tmpfs 1.0M 3.5K 1021K 1% /tmp -/dev/nvme0n1p2 0 0 0 - /boot/efi -/dev/mapper/pve-homelab--staging 19M 4.3K 19M 1% /mnt/staging -/dev/fuse 256K 89 256K 1% /etc/pve -shm 3.9M 1 3.9M 1% /var/lib/containers/storage/overlay-containers/fbb240ea5b547fa140d8edbc3b16daaf4b28a21c7ee424abedab068865a95d5b/userdata/shm -overlay 6.3M 128K 6.2M 2% /var/lib/containers/storage/overlay/4f68ccfa2802b35db2a4c6b396e477f756cb85aa42fa818dd111745ec503616b/merged -tmpfs 1.0K 1 1023 1% /run/credentials/getty@tty1.service -tmpfs 796K 28 796K 1% /run/user/0 -=== STORAGE_BACKENDS === -no pools available -Personalities : -unused devices: -File descriptor 9 (/var/lib/homelab-chat-run/state/runner.lock) leaked on pvs invocation. Parent PID 918816: /usr/bin/bash - /dev/nvme0n1p3 <952.00g 16.00g a-- -File descriptor 9 (/var/lib/homelab-chat-run/state/runner.lock) leaked on vgs invocation. Parent PID 918816: /usr/bin/bash - pve <952.00g 16.00g wz--n- -File descriptor 9 (/var/lib/homelab-chat-run/state/runner.lock) leaked on lvs invocation. Parent PID 918816: /usr/bin/bash - data pve <809.48g twi-aotz-- 33.01 1.42 - [data_tdata] pve <809.48g Twi-ao---- - [data_tmeta] pve <8.26g ewi-ao---- - homelab-staging pve 300.00g Vwi-aotz-- 66.86 - [lvol0_pmspare] pve <8.26g ewi------- - root pve 100.00g -wi-ao---- - snap_vm-150-disk-0_before-nc-freshcheck-20260614-233339 pve 64.00g Vri---tz-k - snap_vm-150-disk-0_nc-posttune-20260614-2343 pve 64.00g Vri---tz-k - swap pve 10.00g -wi-ao---- - vm-110-disk-0 pve 8.00g Vwi-aotz-- 26.63 - vm-112-disk-0 pve 8.00g Vwi-aotz-- 13.54 - vm-150-cloudinit pve 4.00m Vwi-aotz-- 9.38 - vm-150-disk-0 pve 64.00g Vwi-aotz-- 22.72 - vm-170-cloudinit pve 4.00m Vwi-aotz-- 9.38 - vm-170-disk-0 pve 120.00g Vwi-aotz-- 32.00 - vm-171-cloudinit pve 4.00m Vwi-aotz-- 9.38 - vm-171-disk-0 pve 120.00g Vwi-aotz-- 7.05 -=== KERNEL_CRITICAL_LAST_24H === -Jul 20 19:31:56 pve01 systemd-sysv-generator[535545]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 20 19:31:56 pve01 systemd-sysv-generator[535545]: Please update package to include a native systemd unit file. -Jul 20 19:31:56 pve01 systemd-sysv-generator[535545]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 20 19:31:56 pve01 systemd-sysv-generator[535601]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 20 19:31:56 pve01 systemd-sysv-generator[535601]: Please update package to include a native systemd unit file. -Jul 20 19:31:56 pve01 systemd-sysv-generator[535601]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 20 19:31:57 pve01 systemd-sysv-generator[535675]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 20 19:31:57 pve01 systemd-sysv-generator[535675]: Please update package to include a native systemd unit file. -Jul 20 19:31:57 pve01 systemd-sysv-generator[535675]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 20 19:31:57 pve01 systemd-sysv-generator[535733]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 20 19:31:57 pve01 systemd-sysv-generator[535733]: Please update package to include a native systemd unit file. -Jul 20 19:31:57 pve01 systemd-sysv-generator[535733]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 20 22:15:38 pve01 systemd-sysv-generator[1097452]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 20 22:15:38 pve01 systemd-sysv-generator[1097452]: Please update package to include a native systemd unit file. -Jul 20 22:15:38 pve01 systemd-sysv-generator[1097452]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 20 22:15:39 pve01 systemd-sysv-generator[1097526]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 20 22:15:39 pve01 systemd-sysv-generator[1097526]: Please update package to include a native systemd unit file. -Jul 20 22:15:39 pve01 systemd-sysv-generator[1097526]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 20 23:59:14 pve01 systemd-sysv-generator[1457483]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 20 23:59:14 pve01 systemd-sysv-generator[1457483]: Please update package to include a native systemd unit file. -Jul 20 23:59:14 pve01 systemd-sysv-generator[1457483]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 20 23:59:14 pve01 systemd-sysv-generator[1457536]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 20 23:59:14 pve01 systemd-sysv-generator[1457536]: Please update package to include a native systemd unit file. -Jul 20 23:59:14 pve01 systemd-sysv-generator[1457536]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 00:09:32 pve01 systemd-sysv-generator[1497895]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 00:09:32 pve01 systemd-sysv-generator[1497895]: Please update package to include a native systemd unit file. -Jul 21 00:09:32 pve01 systemd-sysv-generator[1497895]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 00:09:32 pve01 systemd-sysv-generator[1497977]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 00:09:32 pve01 systemd-sysv-generator[1497977]: Please update package to include a native systemd unit file. -Jul 21 00:09:32 pve01 systemd-sysv-generator[1497977]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 00:33:12 pve01 systemd-sysv-generator[1580930]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 00:33:12 pve01 systemd-sysv-generator[1580930]: Please update package to include a native systemd unit file. -Jul 21 00:33:12 pve01 systemd-sysv-generator[1580930]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 00:33:12 pve01 systemd-sysv-generator[1580983]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 00:33:12 pve01 systemd-sysv-generator[1580983]: Please update package to include a native systemd unit file. -Jul 21 00:33:12 pve01 systemd-sysv-generator[1580983]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917427]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917427]: Please update package to include a native systemd unit file. -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917427]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917497]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917497]: Please update package to include a native systemd unit file. -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917497]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917569]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917569]: Please update package to include a native systemd unit file. -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917569]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917640]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917640]: Please update package to include a native systemd unit file. -Jul 21 02:10:30 pve01 systemd-sysv-generator[1917640]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:11:53 pve01 systemd-sysv-generator[1930242]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:11:53 pve01 systemd-sysv-generator[1930242]: Please update package to include a native systemd unit file. -Jul 21 02:11:53 pve01 systemd-sysv-generator[1930242]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:11:54 pve01 systemd-sysv-generator[1930305]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:11:54 pve01 systemd-sysv-generator[1930305]: Please update package to include a native systemd unit file. -Jul 21 02:11:54 pve01 systemd-sysv-generator[1930305]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:11:54 pve01 systemd-sysv-generator[1930369]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:11:54 pve01 systemd-sysv-generator[1930369]: Please update package to include a native systemd unit file. -Jul 21 02:11:54 pve01 systemd-sysv-generator[1930369]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:14:02 pve01 systemd-sysv-generator[1932245]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:14:02 pve01 systemd-sysv-generator[1932245]: Please update package to include a native systemd unit file. -Jul 21 02:14:02 pve01 systemd-sysv-generator[1932245]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:14:03 pve01 systemd-sysv-generator[1932308]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:14:03 pve01 systemd-sysv-generator[1932308]: Please update package to include a native systemd unit file. -Jul 21 02:14:03 pve01 systemd-sysv-generator[1932308]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:14:03 pve01 systemd-sysv-generator[1932371]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:14:03 pve01 systemd-sysv-generator[1932371]: Please update package to include a native systemd unit file. -Jul 21 02:14:03 pve01 systemd-sysv-generator[1932371]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:14:04 pve01 systemd-sysv-generator[1932629]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:14:04 pve01 systemd-sysv-generator[1932629]: Please update package to include a native systemd unit file. -Jul 21 02:14:04 pve01 systemd-sysv-generator[1932629]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:14:11 pve01 systemd-sysv-generator[1932867]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:14:11 pve01 systemd-sysv-generator[1932867]: Please update package to include a native systemd unit file. -Jul 21 02:14:11 pve01 systemd-sysv-generator[1932867]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:14:13 pve01 systemd-sysv-generator[1932964]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:14:13 pve01 systemd-sysv-generator[1932964]: Please update package to include a native systemd unit file. -Jul 21 02:14:13 pve01 systemd-sysv-generator[1932964]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:16:57 pve01 systemd-sysv-generator[1955003]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:16:57 pve01 systemd-sysv-generator[1955003]: Please update package to include a native systemd unit file. -Jul 21 02:16:57 pve01 systemd-sysv-generator[1955003]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:16:57 pve01 systemd-sysv-generator[1955071]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:16:57 pve01 systemd-sysv-generator[1955071]: Please update package to include a native systemd unit file. -Jul 21 02:16:57 pve01 systemd-sysv-generator[1955071]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:16:57 pve01 systemd-sysv-generator[1955141]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:16:57 pve01 systemd-sysv-generator[1955141]: Please update package to include a native systemd unit file. -Jul 21 02:16:57 pve01 systemd-sysv-generator[1955141]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:20:11 pve01 systemd-sysv-generator[1957829]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:20:11 pve01 systemd-sysv-generator[1957829]: Please update package to include a native systemd unit file. -Jul 21 02:20:11 pve01 systemd-sysv-generator[1957829]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:20:11 pve01 systemd-sysv-generator[1957891]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:20:11 pve01 systemd-sysv-generator[1957891]: Please update package to include a native systemd unit file. -Jul 21 02:20:11 pve01 systemd-sysv-generator[1957891]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:20:12 pve01 systemd-sysv-generator[1957959]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:20:12 pve01 systemd-sysv-generator[1957959]: Please update package to include a native systemd unit file. -Jul 21 02:20:12 pve01 systemd-sysv-generator[1957959]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:20:13 pve01 systemd-sysv-generator[1958205]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:20:13 pve01 systemd-sysv-generator[1958205]: Please update package to include a native systemd unit file. -Jul 21 02:20:13 pve01 systemd-sysv-generator[1958205]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:20:20 pve01 systemd-sysv-generator[1958431]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:20:20 pve01 systemd-sysv-generator[1958431]: Please update package to include a native systemd unit file. -Jul 21 02:20:20 pve01 systemd-sysv-generator[1958431]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:20:21 pve01 systemd-sysv-generator[1958526]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:20:21 pve01 systemd-sysv-generator[1958526]: Please update package to include a native systemd unit file. -Jul 21 02:20:21 pve01 systemd-sysv-generator[1958526]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:22:02 pve01 systemd-sysv-generator[1971264]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:22:02 pve01 systemd-sysv-generator[1971264]: Please update package to include a native systemd unit file. -Jul 21 02:22:02 pve01 systemd-sysv-generator[1971264]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:22:02 pve01 systemd-sysv-generator[1971316]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:22:02 pve01 systemd-sysv-generator[1971316]: Please update package to include a native systemd unit file. -Jul 21 02:22:02 pve01 systemd-sysv-generator[1971316]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:22:02 pve01 systemd-sysv-generator[1971377]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:22:02 pve01 systemd-sysv-generator[1971377]: Please update package to include a native systemd unit file. -Jul 21 02:22:02 pve01 systemd-sysv-generator[1971377]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:23:50 pve01 systemd-sysv-generator[1973034]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:23:50 pve01 systemd-sysv-generator[1973034]: Please update package to include a native systemd unit file. -Jul 21 02:23:50 pve01 systemd-sysv-generator[1973034]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:23:50 pve01 systemd-sysv-generator[1973101]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:23:50 pve01 systemd-sysv-generator[1973101]: Please update package to include a native systemd unit file. -Jul 21 02:23:50 pve01 systemd-sysv-generator[1973101]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 02:23:50 pve01 systemd-sysv-generator[1973172]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility. -Jul 21 02:23:50 pve01 systemd-sysv-generator[1973172]: Please update package to include a native systemd unit file. -Jul 21 02:23:50 pve01 systemd-sysv-generator[1973172]: ! This compatibility logic is deprecated, expect removal soon. ! -Jul 21 03:30:02 pve01 kernel: EXT4-fs (dm-15): write access unavailable, skipping orphan cleanup -Jul 21 03:30:10 pve01 kernel: EXT4-fs (dm-15): write access unavailable, skipping orphan cleanup -=== BACKUP_HEALTH === - dev-disk-by\x2did-dm\x2dname\x2dpve\x2dvm\x2d\x2d150\x2d\x2dcloudinit.device loaded active plugged /dev/disk/by-id/dm-name-pve-vm--150--cloudinit - dev-disk-by\x2did-dm\x2dname\x2dpve\x2dvm\x2d\x2d170\x2d\x2dcloudinit.device loaded active plugged /dev/disk/by-id/dm-name-pve-vm--170--cloudinit - dev-disk-by\x2did-dm\x2dname\x2dpve\x2dvm\x2d\x2d171\x2d\x2dcloudinit.device loaded active plugged /dev/disk/by-id/dm-name-pve-vm--171--cloudinit - dev-mapper-pve\x2dvm\x2d\x2d150\x2d\x2dcloudinit.device loaded active plugged /dev/mapper/pve-vm--150--cloudinit - dev-mapper-pve\x2dvm\x2d\x2d170\x2d\x2dcloudinit.device loaded active plugged /dev/mapper/pve-vm--170--cloudinit - dev-mapper-pve\x2dvm\x2d\x2d171\x2d\x2dcloudinit.device loaded active plugged /dev/mapper/pve-vm--171--cloudinit - dev-pve-vm\x2d150\x2dcloudinit.device loaded active plugged /dev/pve/vm-150-cloudinit - dev-pve-vm\x2d170\x2dcloudinit.device loaded active plugged /dev/pve/vm-170-cloudinit - dev-pve-vm\x2d171\x2dcloudinit.device loaded active plugged /dev/pve/vm-171-cloudinit - dpkg-db-backup.service loaded inactive dead Daily dpkg database backup service - filebrowser-backup.service loaded inactive dead File Browser backup - homelab-alertmanager-backup.service loaded inactive dead Alertmanager encrypted two-cloud application backup - homelab-appbackup-dockge-stacks.service loaded inactive dead Homelab app-level backup for dockge-stacks - homelab-appbackup-mariadb-batch.service loaded inactive dead Homelab MariaDB app batch backup - homelab-appbackup-npmplus-edge.service loaded inactive dead Homelab app-level backup for npmplus-edge - homelab-appbackup-postgres-batch.service loaded inactive dead Homelab Postgres app batch backup - homelab-appbackup-sqlite-batch.service loaded inactive dead Homelab SQLite app batch backup - homelab-auto-backup.service loaded inactive dead Homelab automatic full backup with retention - homelab-backup-coverage-matrix.service loaded inactive dead Homelab backup coverage matrix - homelab-backup-freshness.service loaded inactive dead Homelab backup freshness collector - homelab-backup-healthcheck.service loaded inactive dead Homelab backup healthcheck -● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state - homelab-backup-sla-health.service loaded inactive dead Homelab backup freshness SLA health - homelab-beszel-backup.service loaded inactive dead Beszel encrypted two-cloud application backup - homelab-blackbox-exporter-backup.service loaded inactive dead Blackbox Exporter encrypted two-cloud application backup - homelab-cluster-admin-incident-engine-backup.service loaded inactive dead Cluster Admin Incident Engine encrypted two-cloud backup - homelab-crypto-portfolio-backup.service loaded inactive dead Gram Crypto portfolio two-cloud backup - homelab-emergency-backup.service loaded inactive dead Homelab emergency/config backup to two encrypted clouds - homelab-extended-appbackup.service loaded inactive dead Homelab extended P1/P2 appbackup run - homelab-gitea-secondary-backup.service loaded inactive dead Create and copy a verified Gitea backup to pve03 - homelab-legacy-monitoring-backups.service loaded inactive dead Legacy monitoring applications encrypted two-cloud backup batch - homelab-mail-cloud-critical-upload.service loaded inactive dead Homelab Mail Cloud critical/small baseline upload - homelab-mail-cloud-edge-vm-upload.service loaded inactive dead Homelab Mail Cloud edge-vm large chunked upload - homelab-mail-cloud-nextcloud-vm-upload.service loaded inactive dead Homelab Mail Cloud Nextcloud VM vzdump upload - homelab-mail-cloud-restore-drill.service loaded inactive dead Homelab Mail Cloud restore drill - homelab-router-running-config-mail-cloud.service loaded inactive dead Homelab router running-config Mail-cloud backup - homelab-verified-backup-health.service loaded inactive dead Homelab verified backup health evidence - homelab-vm-backup-health.service loaded inactive dead Homelab VM backup health - homelab-vm-backup-policy.service loaded inactive dead Homelab VM backup policy and feasibility health - homelab-vm-full-backup.service loaded inactive dead Full vzdump backup of VM170 and VM171 to offhost and two clouds - homelab-vm-local-dumps-2cloud.service loaded inactive dead Homelab VM/CT existing local dumps two-cloud verifier - homelab-vm-local-dumps-retention.service loaded inactive dead Homelab VM local dumps cloud retention -● homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 - immich-media-backup.service loaded inactive dead Immich media backup to staging - immich-media-offhost-sync.service loaded inactive dead Sync Immich media backups from pve01 to pve02 - memos-backup.service loaded inactive dead Memos data backup to pve01 staging - memos-offhost-sync.service loaded inactive dead Memos off-host backup sync to pve02 - postfix.service loaded active running Postfix Mail Transport Agent (main/default instance) - postfix@-.service loaded inactive dead Postfix Mail Transport Agent (instance -, /etc/postfix-/) - skladchik-reports-monitor-backup-restore-check.service loaded inactive dead Restore-check Skladchik reports monitor cloud backups - skladchik-reports-monitor-backup.service loaded inactive dead Backup Skladchik reports monitor from edge-vm to encrypted clouds - dpkg-db-backup.timer loaded active waiting Daily dpkg database backup timer - filebrowser-backup.timer loaded active waiting Run File Browser backup - homelab-alertmanager-backup.timer loaded active waiting Daily Alertmanager two-cloud backup - homelab-appbackup-dockge-stacks.timer loaded active waiting Daily homelab app-level backup timer for dockge-stacks - homelab-appbackup-mariadb-batch.timer loaded active waiting Daily Homelab MariaDB app batch backup timer - homelab-appbackup-npmplus-edge.timer loaded active waiting Daily homelab app-level backup timer for npmplus-edge - homelab-appbackup-postgres-batch.timer loaded active waiting Daily Homelab Postgres app batch backup timer - homelab-appbackup-sqlite-batch.timer loaded active waiting Daily Homelab SQLite app batch backup timer - homelab-auto-backup.timer loaded active waiting Run homelab automatic full backup weekly - homelab-backup-coverage-matrix.timer loaded active waiting Homelab backup coverage matrix timer - homelab-backup-freshness.timer loaded active waiting Run homelab backup freshness collector daily - homelab-backup-healthcheck.timer loaded active waiting Run homelab backup healthcheck daily - homelab-backup-sla-health.timer loaded active waiting Homelab backup freshness SLA health timer - homelab-beszel-backup.timer loaded active waiting Daily Beszel encrypted two-cloud backup - homelab-blackbox-exporter-backup.timer loaded active waiting Daily Blackbox Exporter encrypted two-cloud backup - homelab-cluster-admin-incident-engine-backup.timer loaded active waiting Daily Cluster Admin Incident Engine two-cloud backup - homelab-crypto-portfolio-backup.timer loaded active waiting Daily Gram Crypto portfolio backup - homelab-emergency-backup.timer loaded active waiting Daily homelab emergency/config backup timer - homelab-extended-appbackup.timer loaded active waiting Homelab extended P1/P2 appbackup timer - homelab-gitea-secondary-backup.timer loaded active waiting Daily verified Gitea backup to pve03 - homelab-legacy-monitoring-backups.timer loaded active waiting Daily legacy monitoring applications two-cloud backup batch - homelab-mail-cloud-critical-upload.timer loaded active waiting Daily Homelab Mail Cloud critical/small baseline upload - homelab-mail-cloud-edge-vm-upload.timer loaded active waiting Weekly Homelab Mail Cloud edge-vm large chunked upload - homelab-mail-cloud-nextcloud-vm-upload.timer loaded active waiting Run Homelab Mail Cloud Nextcloud VM upload daily - homelab-mail-cloud-restore-drill.timer loaded active waiting Weekly Homelab Mail Cloud restore drill - homelab-router-running-config-mail-cloud.timer loaded active waiting Run Homelab router running-config Mail-cloud backup daily - homelab-verified-backup-health.timer loaded active waiting Homelab verified backup health evidence timer - homelab-vm-backup-health.timer loaded active waiting Homelab VM backup health timer - homelab-vm-backup-policy.timer loaded active waiting Homelab VM backup policy and feasibility health timer - homelab-vm-full-backup.timer loaded active waiting Daily full backup timer for VM170 and VM171 - homelab-vm-local-dumps-2cloud.timer loaded active waiting Homelab VM/CT existing local dumps two-cloud verifier timer - homelab-vm-local-dumps-retention.timer loaded active waiting Homelab VM local dumps cloud retention timer - memos-backup.timer loaded active waiting Run Memos data backup daily - memos-offhost-sync.timer loaded active waiting Run Memos off-host backup sync daily - skladchik-reports-monitor-backup-restore-check.timer loaded active waiting Weekly restore-check of Skladchik reports monitor cloud backups - skladchik-reports-monitor-backup.timer loaded active waiting Daily Skladchik reports monitor backup to encrypted clouds -Wed 2026-07-22 00:00:00 MSK 6h Tue 2026-07-21 00:00:14 MSK 17h ago dpkg-db-backup.timer dpkg-db-backup.service -Wed 2026-07-22 00:09:43 MSK 6h Tue 2026-07-21 00:09:32 MSK 17h ago homelab-backup-freshness.timer homelab-backup-freshness.service -Wed 2026-07-22 02:15:00 MSK 8h Tue 2026-07-21 02:15:05 MSK 15h ago homelab-vm-full-backup.timer homelab-vm-full-backup.service -Wed 2026-07-22 03:10:02 MSK 9h Tue 2026-07-21 03:13:14 MSK 14h ago homelab-emergency-backup.timer homelab-emergency-backup.service -Wed 2026-07-22 03:35:00 MSK 9h Tue 2026-07-21 03:35:14 MSK 14h ago filebrowser-backup.timer filebrowser-backup.service -Wed 2026-07-22 03:55:02 MSK 10h Tue 2026-07-21 03:51:25 MSK 13h ago homelab-appbackup-dockge-stacks.timer homelab-appbackup-dockge-stacks.service -Wed 2026-07-22 04:17:29 MSK 10h Tue 2026-07-21 04:19:17 MSK 13h ago homelab-appbackup-npmplus-edge.timer homelab-appbackup-npmplus-edge.service -Wed 2026-07-22 04:26:54 MSK 10h Tue 2026-07-21 04:28:14 MSK 13h ago homelab-gitea-secondary-backup.timer homelab-gitea-secondary-backup.service -Wed 2026-07-22 04:38:04 MSK 10h Tue 2026-07-21 04:34:48 MSK 13h ago skladchik-reports-monitor-backup.timer skladchik-reports-monitor-backup.service -Wed 2026-07-22 05:01:44 MSK 11h Tue 2026-07-21 04:57:14 MSK 12h ago homelab-appbackup-sqlite-batch.timer homelab-appbackup-sqlite-batch.service -Wed 2026-07-22 05:23:56 MSK 11h Tue 2026-07-21 05:17:57 MSK 12h ago homelab-router-running-config-mail-cloud.timer homelab-router-running-config-mail-cloud.service -Wed 2026-07-22 05:27:13 MSK 11h Tue 2026-07-21 05:29:05 MSK 12h ago homelab-appbackup-postgres-batch.timer homelab-appbackup-postgres-batch.service -Wed 2026-07-22 05:47:19 MSK 12h Tue 2026-07-21 05:46:14 MSK 11h ago homelab-appbackup-mariadb-batch.timer homelab-appbackup-mariadb-batch.service -Wed 2026-07-22 06:01:19 MSK 12h Tue 2026-07-21 05:57:30 MSK 11h ago homelab-extended-appbackup.timer homelab-extended-appbackup.service -Wed 2026-07-22 06:27:43 MSK 12h Tue 2026-07-21 06:32:25 MSK 11h ago homelab-crypto-portfolio-backup.timer homelab-crypto-portfolio-backup.service -Wed 2026-07-22 06:38:51 MSK 12h Tue 2026-07-21 06:43:54 MSK 10h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service -Wed 2026-07-22 06:39:22 MSK 12h Tue 2026-07-21 06:40:25 MSK 10h ago homelab-alertmanager-backup.timer homelab-alertmanager-backup.service -Wed 2026-07-22 06:44:16 MSK 13h Tue 2026-07-21 06:40:25 MSK 10h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service -Wed 2026-07-22 06:52:06 MSK 13h Tue 2026-07-21 06:47:47 MSK 10h ago homelab-beszel-backup.timer homelab-beszel-backup.service -Wed 2026-07-22 06:58:58 MSK 13h Tue 2026-07-21 06:57:30 MSK 10h ago homelab-blackbox-exporter-backup.timer homelab-blackbox-exporter-backup.service -Wed 2026-07-22 07:02:27 MSK 13h Tue 2026-07-21 07:05:30 MSK 10h ago homelab-backup-healthcheck.timer homelab-backup-healthcheck.service -Wed 2026-07-22 07:07:14 MSK 13h Tue 2026-07-21 07:18:55 MSK 10h ago homelab-legacy-monitoring-backups.timer homelab-legacy-monitoring-backups.service -Wed 2026-07-22 07:35:06 MSK 13h Tue 2026-07-21 07:36:59 MSK 10h ago homelab-vm-backup-health.timer homelab-vm-backup-health.service -Wed 2026-07-22 07:38:21 MSK 13h Tue 2026-07-21 07:36:14 MSK 10h ago homelab-cluster-admin-incident-engine-backup.timer homelab-cluster-admin-incident-engine-backup.service -Wed 2026-07-22 07:41:49 MSK 14h Tue 2026-07-21 07:53:41 MSK 9h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service -Wed 2026-07-22 08:10:37 MSK 14h Tue 2026-07-21 08:07:25 MSK 9h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service -Wed 2026-07-22 08:37:02 MSK 14h Tue 2026-07-21 08:30:07 MSK 9h ago homelab-vm-backup-policy.timer homelab-vm-backup-policy.service -Wed 2026-07-22 08:40:13 MSK 15h Tue 2026-07-21 08:50:14 MSK 8h ago homelab-backup-coverage-matrix.timer homelab-backup-coverage-matrix.service -Wed 2026-07-22 08:53:36 MSK 15h Tue 2026-07-21 08:41:14 MSK 8h ago memos-backup.timer memos-backup.service -Sun 2026-07-26 03:34:47 MSK 4 days Sun 2026-07-19 03:45:37 MSK 2 days ago homelab-auto-backup.timer homelab-auto-backup.service -Sun 2026-07-26 05:17:43 MSK 4 days Sun 2026-07-19 05:11:25 MSK 2 days ago skladchik-reports-monitor-backup-restore-check.timer skladchik-reports-monitor-backup-restore-check.service -Sun 2026-07-26 09:32:36 MSK 4 days Sun 2026-07-19 09:32:29 MSK 2 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service -Sun 2026-07-26 10:29:15 MSK 4 days Sun 2026-07-19 10:06:07 MSK 2 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service -Mon 2026-07-27 07:34:51 MSK 5 days Mon 2026-07-20 07:25:14 MSK 1 day 10h ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service -Jul 19 20:28:30 pve01 homelab-backup-matrix-refresh[4070223]: MATRIX_STATE_STATUS=FAIL -Jul 19 20:28:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 20:28:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 20:33:30 pve01 homelab-backup-matrix-refresh[4092638]: MATRIX_STATE_STATUS=FAIL -Jul 19 20:33:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 20:33:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 20:38:36 pve01 homelab-backup-matrix-refresh[4106577]: MATRIX_STATE_STATUS=FAIL -Jul 19 20:38:36 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 20:38:36 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 20:43:56 pve01 homelab-backup-matrix-refresh[4109241]: MATRIX_STATE_STATUS=FAIL -Jul 19 20:43:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 20:43:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 20:49:15 pve01 homelab-backup-matrix-refresh[4131689]: MATRIX_STATE_STATUS=FAIL -Jul 19 20:49:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 20:49:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 20:54:15 pve01 homelab-backup-matrix-refresh[4145509]: MATRIX_STATE_STATUS=FAIL -Jul 19 20:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 20:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 20:59:15 pve01 homelab-backup-matrix-refresh[4159482]: MATRIX_STATE_STATUS=FAIL -Jul 19 20:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 20:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:04:15 pve01 homelab-backup-matrix-refresh[4181782]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:09:15 pve01 homelab-backup-matrix-refresh[1993]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:14:15 pve01 homelab-backup-matrix-refresh[16108]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:19:15 pve01 homelab-backup-matrix-refresh[38546]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:24:15 pve01 homelab-backup-matrix-refresh[52499]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:29:15 pve01 homelab-backup-matrix-refresh[66476]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:34:15 pve01 homelab-backup-matrix-refresh[88886]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:39:15 pve01 homelab-backup-matrix-refresh[102884]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:44:15 pve01 homelab-backup-matrix-refresh[116812]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:49:26 pve01 homelab-backup-matrix-refresh[139246]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:49:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:49:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:54:30 pve01 homelab-backup-matrix-refresh[153291]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 21:59:30 pve01 homelab-backup-matrix-refresh[167280]: MATRIX_STATE_STATUS=FAIL -Jul 19 21:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 21:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:04:30 pve01 homelab-backup-matrix-refresh[189663]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:09:30 pve01 homelab-backup-matrix-refresh[192354]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:14:30 pve01 homelab-backup-matrix-refresh[206349]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:19:30 pve01 homelab-backup-matrix-refresh[228779]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:24:30 pve01 homelab-backup-matrix-refresh[242779]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:29:30 pve01 homelab-backup-matrix-refresh[256597]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:34:30 pve01 homelab-backup-matrix-refresh[279054]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:34:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:34:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:39:30 pve01 homelab-backup-matrix-refresh[293022]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:44:30 pve01 homelab-backup-matrix-refresh[307038]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:49:46 pve01 homelab-backup-matrix-refresh[330036]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:49:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:49:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 22:55:15 pve01 homelab-backup-matrix-refresh[345990]: MATRIX_STATE_STATUS=FAIL -Jul 19 22:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 22:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:00:16 pve01 homelab-backup-matrix-refresh[361536]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:05:15 pve01 homelab-backup-matrix-refresh[385062]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:10:15 pve01 homelab-backup-matrix-refresh[399332]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:15:16 pve01 homelab-backup-matrix-refresh[403605]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:20:15 pve01 homelab-backup-matrix-refresh[427739]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:25:15 pve01 homelab-backup-matrix-refresh[443219]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:30:16 pve01 homelab-backup-matrix-refresh[458795]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:35:15 pve01 homelab-backup-matrix-refresh[482814]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:40:15 pve01 homelab-backup-matrix-refresh[498238]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:45:16 pve01 homelab-backup-matrix-refresh[513836]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:50:26 pve01 homelab-backup-matrix-refresh[538010]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 19 23:55:30 pve01 homelab-backup-matrix-refresh[553603]: MATRIX_STATE_STATUS=FAIL -Jul 19 23:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 19 23:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:00:16 pve01 homelab-backup-matrix-refresh[569004]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:05:16 pve01 homelab-backup-matrix-refresh[593337]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:05:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:05:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:10:16 pve01 homelab-backup-matrix-refresh[609133]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:10:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:10:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:12:16 pve01 homelab-backup-matrix-refresh[610747]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:12:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:12:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:17:16 pve01 homelab-backup-matrix-refresh[635186]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:17:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:17:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:22:17 pve01 homelab-backup-matrix-refresh[650655]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:22:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:22:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:27:26 pve01 homelab-backup-matrix-refresh[666442]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:27:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:27:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:32:30 pve01 homelab-backup-matrix-refresh[690423]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:37:30 pve01 homelab-backup-matrix-refresh[706247]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:42:30 pve01 homelab-backup-matrix-refresh[721727]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:47:30 pve01 homelab-backup-matrix-refresh[745645]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:52:30 pve01 homelab-backup-matrix-refresh[761096]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 00:57:30 pve01 homelab-backup-matrix-refresh[776579]: MATRIX_STATE_STATUS=FAIL -Jul 20 00:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 00:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:02:30 pve01 homelab-backup-matrix-refresh[797728]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:07:30 pve01 homelab-backup-matrix-refresh[804631]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:12:30 pve01 homelab-backup-matrix-refresh[820114]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:17:30 pve01 homelab-backup-matrix-refresh[844105]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:22:30 pve01 homelab-backup-matrix-refresh[859499]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:27:30 pve01 homelab-backup-matrix-refresh[875036]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:32:30 pve01 homelab-backup-matrix-refresh[899054]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:37:30 pve01 homelab-backup-matrix-refresh[914581]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:42:30 pve01 homelab-backup-matrix-refresh[930107]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:47:30 pve01 homelab-backup-matrix-refresh[954079]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:52:56 pve01 homelab-backup-matrix-refresh[969872]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 01:58:08 pve01 homelab-backup-matrix-refresh[985563]: MATRIX_STATE_STATUS=FAIL -Jul 20 01:58:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 01:58:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:03:15 pve01 homelab-backup-matrix-refresh[1009625]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:08:15 pve01 homelab-backup-matrix-refresh[1025172]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:13:15 pve01 homelab-backup-matrix-refresh[1040687]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:13:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:13:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:18:15 pve01 homelab-backup-matrix-refresh[1064737]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:18:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:18:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:23:15 pve01 homelab-backup-matrix-refresh[1080263]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:23:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:23:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:28:16 pve01 homelab-backup-matrix-refresh[1084502]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:28:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:28:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:33:15 pve01 homelab-backup-matrix-refresh[1108551]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:38:15 pve01 homelab-backup-matrix-refresh[1124188]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:43:15 pve01 homelab-backup-matrix-refresh[1139799]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:48:15 pve01 homelab-backup-matrix-refresh[1163841]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:53:15 pve01 homelab-backup-matrix-refresh[1179474]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:53:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:53:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 02:58:16 pve01 homelab-backup-matrix-refresh[1195011]: MATRIX_STATE_STATUS=FAIL -Jul 20 02:58:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 02:58:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:03:16 pve01 homelab-backup-matrix-refresh[1219515]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:03:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:03:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:08:17 pve01 homelab-backup-matrix-refresh[1235611]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:08:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:08:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:09:51 pve01 homelab-backup-matrix-refresh[1237636]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:09:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:09:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:13:42 pve01 homelab-backup-matrix-refresh[1252592]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:13:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:13:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:19:06 pve01 homelab-backup-matrix-refresh[1276835]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:19:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:19:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:24:07 pve01 homelab-backup-matrix-refresh[1292418]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:29:15 pve01 homelab-backup-matrix-refresh[1307915]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:30:00 pve01 pvescheduler[1308929]: INFO: starting new backup job: vzdump 110 111 112 113 130 150 190 --prune-backups 'keep-daily=4' --mode snapshot --mailnotification failure --storage local --compress zstd --quiet 1 -Jul 20 03:34:15 pve01 homelab-backup-matrix-refresh[1332038]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:35:08 pve01 homelab-backup-matrix-refresh[1333178]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:35:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:35:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:40:15 pve01 homelab-backup-matrix-refresh[1348754]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:45:16 pve01 homelab-backup-matrix-refresh[1364258]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:50:16 pve01 homelab-backup-matrix-refresh[1388202]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:52:56 pve01 homelab-backup-matrix-refresh[1390999]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 03:58:15 pve01 homelab-backup-matrix-refresh[1406759]: MATRIX_STATE_STATUS=FAIL -Jul 20 03:58:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 03:58:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:03:15 pve01 homelab-backup-matrix-refresh[1430825]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:08:15 pve01 homelab-backup-matrix-refresh[1446346]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:10:15 pve01 homelab-backup-matrix-refresh[1448294]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 20 04:15:26 pve01 homelab-backup-matrix-refresh[1464728]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:15:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:15:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:20:01 pve01 homelab-backup-matrix-refresh[1488281]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:20:01 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:20:01 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:25:12 pve01 homelab-backup-matrix-refresh[1503745]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:25:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:25:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:25:47 pve01 homelab-backup-matrix-refresh[1505286]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:25:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:25:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:31:12 pve01 homelab-backup-matrix-refresh[1529373]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:31:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:31:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:36:15 pve01 homelab-backup-matrix-refresh[1544983]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:41:15 pve01 homelab-backup-matrix-refresh[1560445]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:46:15 pve01 homelab-backup-matrix-refresh[1584307]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:51:15 pve01 homelab-backup-matrix-refresh[1599773]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 04:56:15 pve01 homelab-backup-matrix-refresh[1615263]: MATRIX_STATE_STATUS=FAIL -Jul 20 04:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 04:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:01:16 pve01 homelab-backup-matrix-refresh[1640460]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:01:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:01:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:03:43 pve01 homelab-backup-matrix-refresh[1644969]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:03:43 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:03:43 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:08:48 pve01 homelab-backup-matrix-refresh[1660506]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:08:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:08:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:13:56 pve01 homelab-backup-matrix-refresh[1675931]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:13:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:13:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:19:07 pve01 homelab-backup-matrix-refresh[1700241]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:19:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:19:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:24:07 pve01 homelab-backup-matrix-refresh[1715665]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:27:56 pve01 homelab-backup-matrix-refresh[1733709]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:27:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:27:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:33:15 pve01 homelab-backup-matrix-refresh[1757822]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:38:15 pve01 homelab-backup-matrix-refresh[1773361]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:43:15 pve01 homelab-backup-matrix-refresh[1788819]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:48:15 pve01 homelab-backup-matrix-refresh[1812619]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:49:50 pve01 homelab-backup-matrix-refresh[1814730]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:49:50 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:49:50 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 05:54:56 pve01 homelab-backup-matrix-refresh[1830204]: MATRIX_STATE_STATUS=FAIL -Jul 20 05:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 05:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:00:16 pve01 homelab-backup-matrix-refresh[1845897]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:05:15 pve01 homelab-backup-matrix-refresh[1872116]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:10:15 pve01 homelab-backup-matrix-refresh[1893683]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:13:16 pve01 homelab-backup-matrix-refresh[1900739]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:13:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:13:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:18:18 pve01 homelab-backup-matrix-refresh[1924679]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:18:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:18:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:23:26 pve01 homelab-backup-matrix-refresh[1940188]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:23:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:23:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:28:26 pve01 homelab-backup-matrix-refresh[1956242]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:28:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:28:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:33:26 pve01 homelab-backup-matrix-refresh[1980145]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:33:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:33:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:35:21 pve01 homelab-backup-matrix-refresh[1984457]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:35:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:35:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:40:26 pve01 homelab-backup-matrix-refresh[2000528]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:45:30 pve01 homelab-backup-matrix-refresh[2016958]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:50:30 pve01 homelab-backup-matrix-refresh[2040093]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 06:52:20 pve01 homelab-mail-cloud-critical-upload[2039699]: REMOTE_FAIL=0 -Jul 20 06:55:30 pve01 homelab-backup-matrix-refresh[2056859]: MATRIX_STATE_STATUS=FAIL -Jul 20 06:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 06:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:00:30 pve01 homelab-backup-matrix-refresh[2073835]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:05:30 pve01 homelab-backup-matrix-refresh[2097030]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:08:29 pve01 homelab-backup-matrix-refresh[2111023]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:08:29 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:08:29 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:13:30 pve01 homelab-backup-matrix-refresh[2123578]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:13:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:13:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:18:30 pve01 homelab-backup-matrix-refresh[2140101]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:18:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:18:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:23:30 pve01 homelab-backup-matrix-refresh[2158207]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:23:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:23:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:25:15 pve01 homelab-backup-matrix-refresh[2171082]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:30:16 pve01 homelab-backup-matrix-refresh[2186510]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:35:15 pve01 homelab-backup-matrix-refresh[2210494]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:40:21 pve01 homelab-backup-matrix-refresh[2226665]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:40:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:40:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:41:30 pve01 homelab-backup-matrix-refresh[2227922]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:46:32 pve01 homelab-backup-matrix-refresh[2251943]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:46:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:46:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:51:56 pve01 homelab-backup-matrix-refresh[2267857]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:51:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:51:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 07:57:08 pve01 homelab-backup-matrix-refresh[2284427]: MATRIX_STATE_STATUS=FAIL -Jul 20 07:57:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 07:57:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:02:16 pve01 homelab-backup-matrix-refresh[2297063]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:02:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:02:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:07:15 pve01 homelab-backup-matrix-refresh[2316882]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:10:25 pve01 homelab-overall-health[2331217]: STATUS=WARN TS=2026-07-20T05:10:25Z TYPE=overall-health BAD=5 backup-framework.txt=OK drift-check.txt=WARN service-registry.txt=OK dependency-map.txt=OK golden-state.txt=OK cluster-passport.txt=WARN final-readiness.txt=WARN safe-autoheal.txt=OK kuma-monitor-policy.txt=OK backup-sla.txt=OK backup-coverage-matrix.txt=OK extended-appbackup.txt=OK residual-review.txt=OK forum-snuffleupagus.txt=OK incident-journal.txt=OK duty-admin-v2.txt=OK node-loss-readiness.txt=OK node-loss-runbooks.txt=OK power-loss-readiness.txt=OK power-loss-runbook.txt=OK ungated-health-backlog.txt=OK pve03-root-cleanup.txt=OK pve03-staging-retention.txt=OK pve03-staging-retention-proof.txt=OK pve03-staging-retention-cleanup.txt=OK pve03-failed-unit-cleanup.txt=OK remote-git-sops-age-readiness.txt=OK remote-git-sops-age-policy.txt=OK desired-state-remote-target-trace.txt=OK desired-state-remote-target.txt=OK desired-state.txt=OK runbooks.txt=OK alerting.txt=OK secret-exposure.txt=OK capacity-risk.txt=OK vm-local-dumps-retention.txt=OK vm-local-dumps-cloud.txt=OK vm-backup-policy.txt=OK vm170-vm171-full-strategy.txt=OK external-probe-vps.txt=OK netbird-vps-trust.txt=OK netbird-vps-trust-closure.txt=OK external-probe-runner-decision.txt=OK netbird-vps-trust-clean-seal.txt=OK live-tail-audit.txt=OK vm-backup.txt=OK cluster-admin-observer.txt=OK cluster-admin-restricted-probes.txt=WARN cluster-admin-full-observer.txt=WARN cluster-admin-vm-mail-cloud-backup.txt=OK cluster-admin-webpanel-sync.txt=OK cluster-admin-webpanel.txt=OK -Jul 20 08:10:27 pve01 homelab-backup-matrix-refresh[2331605]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:10:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:10:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:15:30 pve01 homelab-backup-matrix-refresh[2347983]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:20:30 pve01 homelab-backup-matrix-refresh[2371091]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:25:30 pve01 homelab-backup-matrix-refresh[2386601]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:30:22 pve01 homelab-backup-matrix-refresh[2403498]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:30:22 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:30:22 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:35:26 pve01 homelab-backup-matrix-refresh[2426661]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:35:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:35:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:36:16 pve01 homelab-backup-matrix-refresh[2427543]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:36:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:36:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:41:18 pve01 homelab-backup-matrix-refresh[2443027]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:41:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:41:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:46:26 pve01 homelab-backup-matrix-refresh[2467111]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:46:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:46:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:47:15 pve01 homelab-backup-matrix-refresh[2468144]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:47:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:47:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:52:26 pve01 homelab-backup-matrix-refresh[2484169]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:52:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:52:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 08:57:30 pve01 homelab-backup-matrix-refresh[2499762]: MATRIX_STATE_STATUS=FAIL -Jul 20 08:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 08:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:02:30 pve01 homelab-backup-matrix-refresh[2523793]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:07:30 pve01 homelab-backup-matrix-refresh[2539245]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:12:30 pve01 homelab-backup-matrix-refresh[2554758]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:17:30 pve01 homelab-backup-matrix-refresh[2567370]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:22:30 pve01 homelab-backup-matrix-refresh[2583117]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:27:30 pve01 homelab-backup-matrix-refresh[2598614]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:32:30 pve01 homelab-backup-matrix-refresh[2622549]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:37:30 pve01 homelab-backup-matrix-refresh[2638103]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:42:30 pve01 homelab-backup-matrix-refresh[2653680]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:47:30 pve01 homelab-backup-matrix-refresh[2677702]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:52:30 pve01 homelab-backup-matrix-refresh[2693207]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 09:57:30 pve01 homelab-backup-matrix-refresh[2708789]: MATRIX_STATE_STATUS=FAIL -Jul 20 09:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 09:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:02:30 pve01 homelab-backup-matrix-refresh[2732734]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:04:07 pve01 homelab-backup-matrix-refresh[2734125]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:04:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:04:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:09:07 pve01 homelab-backup-matrix-refresh[2749537]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:09:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:09:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:14:07 pve01 homelab-backup-matrix-refresh[2765091]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:14:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:14:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:19:15 pve01 homelab-backup-matrix-refresh[2789006]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:24:15 pve01 homelab-backup-matrix-refresh[2804462]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:29:15 pve01 homelab-backup-matrix-refresh[2819907]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:34:15 pve01 homelab-backup-matrix-refresh[2843900]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:39:15 pve01 homelab-backup-matrix-refresh[2859366]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:44:16 pve01 homelab-backup-matrix-refresh[2863481]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:44:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:44:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:49:15 pve01 homelab-backup-matrix-refresh[2887434]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:54:15 pve01 homelab-backup-matrix-refresh[2902771]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 10:59:15 pve01 homelab-backup-matrix-refresh[2918259]: MATRIX_STATE_STATUS=FAIL -Jul 20 10:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 10:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:04:15 pve01 homelab-backup-matrix-refresh[2942234]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:09:15 pve01 homelab-backup-matrix-refresh[2957726]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:14:15 pve01 homelab-backup-matrix-refresh[2973175]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:19:15 pve01 homelab-backup-matrix-refresh[2997114]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:24:15 pve01 homelab-backup-matrix-refresh[3012539]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:29:15 pve01 homelab-backup-matrix-refresh[3028098]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:34:26 pve01 homelab-backup-matrix-refresh[3052177]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:34:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:34:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:39:30 pve01 homelab-backup-matrix-refresh[3067593]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:44:30 pve01 homelab-backup-matrix-refresh[3083139]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:49:30 pve01 homelab-backup-matrix-refresh[3107024]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:49:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:49:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:54:30 pve01 homelab-backup-matrix-refresh[3122507]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 11:59:30 pve01 homelab-backup-matrix-refresh[3137990]: MATRIX_STATE_STATUS=FAIL -Jul 20 11:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 11:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:04:30 pve01 homelab-backup-matrix-refresh[3150598]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:09:30 pve01 homelab-backup-matrix-refresh[3166051]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:14:30 pve01 homelab-backup-matrix-refresh[3181454]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:19:30 pve01 homelab-backup-matrix-refresh[3205349]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:24:30 pve01 homelab-backup-matrix-refresh[3220920]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:29:30 pve01 homelab-backup-matrix-refresh[3236455]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:34:56 pve01 homelab-backup-matrix-refresh[3260683]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:34:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:34:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:40:08 pve01 homelab-backup-matrix-refresh[3276379]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:40:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:40:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:45:09 pve01 homelab-backup-matrix-refresh[3297387]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:45:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:45:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:50:15 pve01 homelab-backup-matrix-refresh[3315975]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 12:55:15 pve01 homelab-backup-matrix-refresh[3331521]: MATRIX_STATE_STATUS=FAIL -Jul 20 12:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 12:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:00:15 pve01 homelab-backup-matrix-refresh[3347832]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:05:15 pve01 homelab-backup-matrix-refresh[3370899]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:10:15 pve01 homelab-backup-matrix-refresh[3386358]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:15:16 pve01 homelab-backup-matrix-refresh[3401894]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:20:15 pve01 homelab-backup-matrix-refresh[3425830]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:25:15 pve01 homelab-backup-matrix-refresh[3429905]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:30:15 pve01 homelab-backup-matrix-refresh[3446239]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:30:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:30:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:35:15 pve01 homelab-backup-matrix-refresh[3469372]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:40:15 pve01 homelab-backup-matrix-refresh[3484818]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:45:16 pve01 homelab-backup-matrix-refresh[3500319]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:50:26 pve01 homelab-backup-matrix-refresh[3524417]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 13:55:30 pve01 homelab-backup-matrix-refresh[3539958]: MATRIX_STATE_STATUS=FAIL -Jul 20 13:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 13:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:00:30 pve01 homelab-backup-matrix-refresh[3556311]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:05:30 pve01 homelab-backup-matrix-refresh[3579422]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:10:30 pve01 homelab-backup-matrix-refresh[3594909]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:10:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:10:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:15:30 pve01 homelab-backup-matrix-refresh[3618728]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:20:30 pve01 homelab-backup-matrix-refresh[3634183]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:25:30 pve01 homelab-backup-matrix-refresh[3649674]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:30:30 pve01 homelab-backup-matrix-refresh[3673515]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:30:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:30:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:35:30 pve01 homelab-backup-matrix-refresh[3689095]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:35:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:35:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:40:31 pve01 homelab-backup-matrix-refresh[3698580]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:45:30 pve01 homelab-backup-matrix-refresh[3709574]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:50:30 pve01 homelab-backup-matrix-refresh[3732500]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 14:55:30 pve01 homelab-backup-matrix-refresh[3747946]: MATRIX_STATE_STATUS=FAIL -Jul 20 14:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 14:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:00:30 pve01 homelab-backup-matrix-refresh[3771782]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:05:48 pve01 homelab-backup-matrix-refresh[3787552]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:05:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:05:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:11:06 pve01 homelab-backup-matrix-refresh[3803194]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:11:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:11:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:16:16 pve01 homelab-backup-matrix-refresh[3819582]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:16:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:16:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:21:15 pve01 homelab-backup-matrix-refresh[3842674]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:26:15 pve01 homelab-backup-matrix-refresh[3858134]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:26:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:26:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:31:15 pve01 homelab-backup-matrix-refresh[3874469]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:36:15 pve01 homelab-backup-matrix-refresh[3897548]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:41:15 pve01 homelab-backup-matrix-refresh[3912924]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:46:16 pve01 homelab-backup-matrix-refresh[3917964]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:46:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:46:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:51:15 pve01 homelab-backup-matrix-refresh[3941009]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 15:56:15 pve01 homelab-backup-matrix-refresh[3956553]: MATRIX_STATE_STATUS=FAIL -Jul 20 15:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 15:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:01:15 pve01 homelab-backup-matrix-refresh[3972944]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:01:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:01:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:06:15 pve01 homelab-backup-matrix-refresh[3996047]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:06:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:06:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:11:15 pve01 homelab-backup-matrix-refresh[4011569]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:16:15 pve01 homelab-backup-matrix-refresh[4035432]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:16:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:16:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:21:15 pve01 homelab-backup-matrix-refresh[4050883]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:26:16 pve01 homelab-backup-matrix-refresh[4066337]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:26:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:26:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:31:18 pve01 homelab-backup-matrix-refresh[4090328]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:31:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:31:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:36:26 pve01 homelab-backup-matrix-refresh[4106097]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:36:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:36:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:41:30 pve01 homelab-backup-matrix-refresh[4121654]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:46:30 pve01 homelab-backup-matrix-refresh[4145630]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:51:30 pve01 homelab-backup-matrix-refresh[4161061]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 16:56:31 pve01 homelab-backup-matrix-refresh[4176634]: MATRIX_STATE_STATUS=FAIL -Jul 20 16:56:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 16:56:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:01:30 pve01 homelab-backup-matrix-refresh[6881]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:06:30 pve01 homelab-backup-matrix-refresh[22575]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:11:30 pve01 homelab-backup-matrix-refresh[38083]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:16:31 pve01 homelab-backup-matrix-refresh[59658]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:21:30 pve01 homelab-backup-matrix-refresh[66183]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:26:30 pve01 homelab-backup-matrix-refresh[81598]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:31:31 pve01 homelab-backup-matrix-refresh[105617]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:36:30 pve01 homelab-backup-matrix-refresh[121433]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:36:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:36:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:41:30 pve01 homelab-backup-matrix-refresh[136937]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:46:30 pve01 homelab-backup-matrix-refresh[160897]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:51:30 pve01 homelab-backup-matrix-refresh[176488]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 17:56:30 pve01 homelab-backup-matrix-refresh[191911]: MATRIX_STATE_STATUS=FAIL -Jul 20 17:56:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 17:56:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:01:30 pve01 homelab-backup-matrix-refresh[215914]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:06:30 pve01 homelab-backup-matrix-refresh[231465]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:11:30 pve01 homelab-backup-matrix-refresh[247009]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:16:31 pve01 homelab-backup-matrix-refresh[270971]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:21:30 pve01 homelab-backup-matrix-refresh[286516]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:26:30 pve01 homelab-backup-matrix-refresh[290662]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:31:30 pve01 homelab-backup-matrix-refresh[314846]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:36:46 pve01 homelab-backup-matrix-refresh[330621]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:36:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:36:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:41:48 pve01 homelab-backup-matrix-refresh[346168]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:41:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:41:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:46:58 pve01 homelab-backup-matrix-refresh[370235]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:46:58 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:46:58 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:52:08 pve01 homelab-backup-matrix-refresh[385867]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:52:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:52:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 18:57:15 pve01 homelab-backup-matrix-refresh[401443]: MATRIX_STATE_STATUS=FAIL -Jul 20 18:57:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 18:57:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:02:15 pve01 homelab-backup-matrix-refresh[425446]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:02:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:02:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:07:15 pve01 homelab-backup-matrix-refresh[440954]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:12:15 pve01 homelab-backup-matrix-refresh[456367]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:12:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:12:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:17:15 pve01 homelab-backup-matrix-refresh[480328]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:17:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:17:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:22:15 pve01 homelab-backup-matrix-refresh[495871]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:22:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:22:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:27:15 pve01 homelab-backup-matrix-refresh[511358]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:27:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:27:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:28:51 pve01 homelab-backup-matrix-refresh[512760]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:28:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:28:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 20 19:34:08 pve01 homelab-backup-matrix-refresh[537480]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:34:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:34:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:39:09 pve01 homelab-backup-matrix-refresh[553115]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:39:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:39:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:44:15 pve01 homelab-backup-matrix-refresh[568668]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:49:15 pve01 homelab-backup-matrix-refresh[592723]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:54:15 pve01 homelab-backup-matrix-refresh[608272]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 19:59:15 pve01 homelab-backup-matrix-refresh[623767]: MATRIX_STATE_STATUS=FAIL -Jul 20 19:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 19:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:04:15 pve01 homelab-backup-matrix-refresh[647775]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:09:15 pve01 homelab-backup-matrix-refresh[663153]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:14:15 pve01 homelab-backup-matrix-refresh[667315]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:19:26 pve01 homelab-backup-matrix-refresh[691333]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:19:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:19:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:24:31 pve01 homelab-backup-matrix-refresh[706901]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:24:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:24:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:29:31 pve01 homelab-backup-matrix-refresh[722368]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:29:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:29:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:34:31 pve01 homelab-backup-matrix-refresh[746391]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:34:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:34:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:39:31 pve01 homelab-backup-matrix-refresh[761801]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:39:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:39:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:44:31 pve01 homelab-backup-matrix-refresh[777273]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:44:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:44:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:49:31 pve01 homelab-backup-matrix-refresh[801193]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:49:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:49:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 20:54:56 pve01 homelab-backup-matrix-refresh[816954]: MATRIX_STATE_STATUS=FAIL -Jul 20 20:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 20:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:00:16 pve01 homelab-backup-matrix-refresh[832684]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:05:15 pve01 homelab-backup-matrix-refresh[856679]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:10:15 pve01 homelab-backup-matrix-refresh[872167]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:15:16 pve01 homelab-backup-matrix-refresh[887656]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:20:15 pve01 homelab-backup-matrix-refresh[911605]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:25:16 pve01 homelab-backup-matrix-refresh[915711]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:25:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:25:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:30:16 pve01 homelab-backup-matrix-refresh[931433]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:35:16 pve01 homelab-backup-matrix-refresh[955099]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:40:26 pve01 homelab-backup-matrix-refresh[970563]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:45:28 pve01 homelab-backup-matrix-refresh[986962]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:45:28 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:45:28 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:50:30 pve01 homelab-backup-matrix-refresh[1009969]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 21:55:30 pve01 homelab-backup-matrix-refresh[1025505]: MATRIX_STATE_STATUS=FAIL -Jul 20 21:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 21:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:00:31 pve01 homelab-backup-matrix-refresh[1041885]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:05:31 pve01 homelab-backup-matrix-refresh[1065096]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:10:31 pve01 homelab-backup-matrix-refresh[1080543]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:15:31 pve01 homelab-backup-matrix-refresh[1096922]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:15:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:15:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:20:31 pve01 homelab-backup-matrix-refresh[1120354]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:20:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:20:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:25:31 pve01 homelab-backup-matrix-refresh[1135777]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:25:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:25:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:30:31 pve01 homelab-backup-matrix-refresh[1152095]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:30:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:30:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:35:31 pve01 homelab-backup-matrix-refresh[1175243]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:35:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:35:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:40:31 pve01 homelab-backup-matrix-refresh[1179415]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:45:31 pve01 homelab-backup-matrix-refresh[1195757]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:45:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:45:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:50:31 pve01 homelab-backup-matrix-refresh[1218911]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:50:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:50:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 22:55:31 pve01 homelab-backup-matrix-refresh[1234331]: MATRIX_STATE_STATUS=FAIL -Jul 20 22:55:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 22:55:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:00:31 pve01 homelab-backup-matrix-refresh[1258173]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:05:31 pve01 homelab-backup-matrix-refresh[1273772]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:10:31 pve01 homelab-backup-matrix-refresh[1289212]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:15:32 pve01 homelab-backup-matrix-refresh[1313105]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:15:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:15:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:20:48 pve01 homelab-backup-matrix-refresh[1328836]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:20:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:20:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:25:49 pve01 homelab-backup-matrix-refresh[1344339]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:25:49 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:25:49 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:31:15 pve01 homelab-backup-matrix-refresh[1360964]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:36:15 pve01 homelab-backup-matrix-refresh[1384068]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:41:15 pve01 homelab-backup-matrix-refresh[1399485]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:46:15 pve01 homelab-backup-matrix-refresh[1423238]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:51:16 pve01 homelab-backup-matrix-refresh[1427834]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:51:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:51:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 20 23:56:15 pve01 homelab-backup-matrix-refresh[1443462]: MATRIX_STATE_STATUS=FAIL -Jul 20 23:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 20 23:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:00:16 pve01 homelab-backup-matrix-refresh[1458364]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:05:26 pve01 homelab-backup-matrix-refresh[1482896]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:05:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:05:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:09:33 pve01 homelab-backup-matrix-refresh[1498014]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:09:33 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:09:33 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:09:34 pve01 homelab-safe-exec[1498016]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -Jul 21 00:10:55 pve01 homelab-safe-exec[1499369]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -Jul 21 00:10:55 pve01 homelab-safe-exec[1499369]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -Jul 21 00:14:48 pve01 homelab-backup-matrix-refresh[1513957]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:14:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:14:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:20:00 pve01 homelab-backup-matrix-refresh[1538116]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:20:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:20:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:25:04 pve01 homelab-backup-matrix-refresh[1553654]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:25:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:25:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:26:08 pve01 homelab-safe-exec[1554627]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -Jul 21 00:26:08 pve01 homelab-safe-exec[1554627]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -Jul 21 00:30:05 pve01 homelab-backup-matrix-refresh[1569592]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:30:05 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:30:05 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:35:16 pve01 homelab-backup-matrix-refresh[1594059]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:40:15 pve01 homelab-backup-matrix-refresh[1609512]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:42:17 pve01 homelab-safe-exec[1611283]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -Jul 21 00:42:17 pve01 homelab-safe-exec[1611283]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -Jul 21 00:45:15 pve01 homelab-backup-matrix-refresh[1633504]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:45:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:45:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:50:15 pve01 homelab-backup-matrix-refresh[1637691]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:55:15 pve01 homelab-backup-matrix-refresh[1653171]: MATRIX_STATE_STATUS=FAIL -Jul 21 00:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 00:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 00:58:17 pve01 homelab-safe-exec[1667105]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -Jul 21 00:58:17 pve01 homelab-safe-exec[1667105]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -Jul 21 01:00:15 pve01 homelab-backup-matrix-refresh[1669525]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:05:15 pve01 homelab-backup-matrix-refresh[1692700]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:10:15 pve01 homelab-backup-matrix-refresh[1708083]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:15:16 pve01 homelab-backup-matrix-refresh[1723575]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:15:18 pve01 homelab-safe-exec[1723574]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -Jul 21 01:20:15 pve01 homelab-backup-matrix-refresh[1747690]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:25:15 pve01 homelab-backup-matrix-refresh[1763279]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:30:15 pve01 homelab-backup-matrix-refresh[1778809]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:30:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:30:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:32:09 pve01 homelab-safe-exec[1789005]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -Jul 21 01:32:09 pve01 homelab-safe-exec[1789005]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -Jul 21 01:35:15 pve01 homelab-backup-matrix-refresh[1802971]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:40:15 pve01 homelab-backup-matrix-refresh[1818394]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:45:15 pve01 homelab-backup-matrix-refresh[1834787]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:45:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:45:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:48:17 pve01 homelab-safe-exec[1844870]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -Jul 21 01:48:17 pve01 homelab-safe-exec[1844870]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -Jul 21 01:50:15 pve01 homelab-backup-matrix-refresh[1857955]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 01:55:26 pve01 homelab-backup-matrix-refresh[1873552]: MATRIX_STATE_STATUS=FAIL -Jul 21 01:55:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 01:55:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 02:00:26 pve01 homelab-backup-matrix-refresh[1878517]: MATRIX_STATE_STATUS=FAIL -Jul 21 02:00:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:00:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 02:03:27 pve01 homelab-safe-exec[1900187]: homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -Jul 21 02:03:27 pve01 homelab-safe-exec[1900187]: homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 -Jul 21 02:05:31 pve01 homelab-backup-matrix-refresh[1901804]: MATRIX_STATE_STATUS=FAIL -Jul 21 02:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 02:10:31 pve01 homelab-backup-matrix-refresh[1917432]: MATRIX_STATE_STATUS=FAIL -Jul 21 02:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 02:14:04 pve01 homelab-backup-matrix-refresh[1932401]: MATRIX_STATE_STATUS=FAIL -Jul 21 02:14:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:14:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 02:14:13 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:14:13 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 21 02:20:13 pve01 homelab-backup-matrix-refresh[1957976]: MATRIX_STATE_STATUS=FAIL -Jul 21 02:20:13 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:20:13 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 02:20:21 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:20:21 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 21 02:33:04 pve01 homelab-backup-matrix-refresh[2011530]: MATRIX_STATE_STATUS=FAIL -Jul 21 02:33:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:33:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'. -Jul 21 03:09:44 pve01 homelab-backup-matrix-refresh[2136275]: MATRIX_STATE_STATUS=FAIL -Jul 21 03:09:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 03:09:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 03:13:44 pve01 homelab-backup-matrix-refresh[2138763]: MATRIX_STATE_STATUS=FAIL -Jul 21 03:13:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 03:13:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 03:30:02 pve01 pvescheduler[2189056]: INFO: starting new backup job: vzdump 110 111 112 113 130 150 190 --compress zstd --quiet 1 --mode snapshot --prune-backups 'keep-daily=4' --mailnotification failure --storage local -Jul 21 03:35:15 pve01 homelab-backup-matrix-refresh[2211416]: MATRIX_STATE_STATUS=FAIL -Jul 21 03:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 03:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 03:51:56 pve01 homelab-backup-matrix-refresh[2262529]: MATRIX_STATE_STATUS=FAIL -Jul 21 03:51:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 03:51:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 04:10:08 pve01 homelab-backup-matrix-refresh[2324930]: MATRIX_STATE_STATUS=FAIL -Jul 21 04:10:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 04:10:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 04:19:51 pve01 homelab-backup-matrix-refresh[2361594]: MATRIX_STATE_STATUS=FAIL -Jul 21 04:19:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 04:19:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 04:35:23 pve01 homelab-backup-matrix-refresh[2412406]: MATRIX_STATE_STATUS=FAIL -Jul 21 04:35:23 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 04:35:23 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 05:00:46 pve01 homelab-backup-matrix-refresh[2482684]: MATRIX_STATE_STATUS=FAIL -Jul 21 05:00:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 05:00:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 05:32:33 pve01 appbackupctl[2593535]: Failed to lsf with 2 errors: last error was: error in ListJSON: file error 400 () -Jul 21 05:32:39 pve01 homelab-backup-matrix-refresh[2593863]: MATRIX_STATE_STATUS=FAIL -Jul 21 05:32:39 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 05:32:39 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_STATUS=FAIL -Jul 21 05:46:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 05:46:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_STATUS=FAIL -Jul 21 06:06:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 06:06:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 06:42:44 pve01 homelab-mail-cloud-critical-upload[2820985]: REMOTE_FAIL=0 -Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_STATUS=FAIL -Jul 21 06:45:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 06:45:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_STATUS=FAIL -Jul 21 07:05:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 07:05:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_STATUS=FAIL -Jul 21 07:37:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 07:37:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 08:01:15 pve01 homelab-overall-health[3098671]: STATUS=WARN TS=2026-07-21T05:01:14Z TYPE=overall-health BAD=7 backup-framework.txt=OK drift-check.txt=WARN service-registry.txt=OK dependency-map.txt=OK golden-state.txt=OK cluster-passport.txt=WARN final-readiness.txt=WARN safe-autoheal.txt=OK kuma-monitor-policy.txt=OK backup-sla.txt=WARN backup-coverage-matrix.txt=OK extended-appbackup.txt=OK residual-review.txt=OK forum-snuffleupagus.txt=OK incident-journal.txt=OK duty-admin-v2.txt=OK node-loss-readiness.txt=OK node-loss-runbooks.txt=OK power-loss-readiness.txt=OK power-loss-runbook.txt=OK ungated-health-backlog.txt=OK pve03-root-cleanup.txt=OK pve03-staging-retention.txt=OK pve03-staging-retention-proof.txt=OK pve03-staging-retention-cleanup.txt=OK pve03-failed-unit-cleanup.txt=OK remote-git-sops-age-readiness.txt=OK remote-git-sops-age-policy.txt=OK desired-state-remote-target-trace.txt=OK desired-state-remote-target.txt=OK desired-state.txt=OK runbooks.txt=OK alerting.txt=OK secret-exposure.txt=OK capacity-risk.txt=OK vm-local-dumps-retention.txt=OK vm-local-dumps-cloud.txt=OK vm-backup-policy.txt=OK vm170-vm171-full-strategy.txt=OK external-probe-vps.txt=OK netbird-vps-trust.txt=OK netbird-vps-trust-closure.txt=OK external-probe-runner-decision.txt=OK netbird-vps-trust-clean-seal.txt=OK live-tail-audit.txt=OK vm-backup.txt=OK cluster-admin-observer.txt=WARN cluster-admin-restricted-probes.txt=WARN cluster-admin-full-observer.txt=WARN cluster-admin-vm-mail-cloud-backup.txt=OK cluster-admin-webpanel-sync.txt=OK cluster-admin-webpanel.txt=OK -Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_STATUS=FAIL -Jul 21 08:07:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 08:07:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_STATUS=FAIL -Jul 21 08:30:14 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 08:30:14 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_STATUS=FAIL -Jul 21 08:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 08:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_STATUS=FAIL -Jul 21 08:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 08:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_STATUS=FAIL -Jul 21 10:02:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -Jul 21 10:02:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -CRITICAL_FIRST_DIAGNOSTIC=COMPLETE +=== DECISION_COUNTS === +QUORUM_RISKS=0 +BACKUP_RISKS=1202 +KERNEL_RISKS=0 +FAILED_UNIT_RISKS=6 +CAPACITY_RISKS=0 +=== NEXT_ACTION === +PRIORITY=P0_BACKUP_POLICY +EVIDENCE=● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state +CRITICAL_DIAGNOSTIC_DECISION=COMPLETE OUTPUT_END CHAT_OUTPUT_END