CHAT_OUTPUT_BEGIN
COMMAND_ID=CRITICAL-FIRST-DIAGNOSTIC-103
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=cluster-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=32ad0722a84aebeb4f1e33dab07a771683458bec2422895b557411be769cd3f2
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=b23c2f575c56d419a6dfb673cbf42ce15bc9a9c8e2892d3d6a6f9a91c9a7a3ce
SANITIZED_OUTPUT_SHA256=2a82a08aa78353751e68781b74d3029143cd010fb8931ff0b43efc0b8686b853
OUTPUT_BEGIN
=== VERIFIED_PREVIOUS_RESULT ===
COMMAND_ID=CRITICAL-FIRST-ACTION-102
IMMUTABLE_COMMIT=4cf04bbfdae66e66e09e451a6a5eae4abb874704
cut: '"': No such file or directory
JSON_SHA256=
cut: '"': No such file or directory
TXT_SHA256=
PREVIOUS_COMMAND_IN_ERROR_REGISTER=no
=== FIRST_PRIORITY_FINDINGS ===
=== FINDING_COUNTS ===
P0_MATCHES=45
P1_MATCHES=87
STORAGE_RISK_MATCHES=2
RECENT_FAILED_COMMAND_LINES=0
BOOTSTRAP_SECTION_LINES=0
=== ACTIONABLE_P0 ===
=== ACTIONABLE_P0 ===
86:398:- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained.
155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.
203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.
486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
546:● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state
548:● homelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03
1102:/var/lib/homelab-health/backup-matrix-cluster-final-seal.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
1104:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-production-closure.txt:1:STATUS=OK TS=2026-07-13T08:12:14Z TYPE=crypto-portfolio-ui-center-alert-currency-production-closure BAD=0 PRODUCTION_CHANGED=YES_UI_ONLY INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 POSTCHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION_ROLLBACK_COUNT=3 RETENTION_OPTIONAL_OTHER_CANDIDATES=1 RETENTION_REMOVED_ROLLBACK_COUNT=0 DANGLING_IMAGES_REMAINING=2 UNREFERENCED_DANGLING_IMAGES=0 DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE_CLEAN=YES PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES READY_FOR_UI_FINAL_SEAL=YES
1106:/var/lib/homelab-health/cluster-admin-stage3-postapply-backup.txt:1:STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z
1109:/var/lib/homelab-health/pve01-core-monitoring-app-restorecheck.txt:1:STATUS=OK TS=2026-07-08T15:57:27Z TYPE=pve01-core-monitoring-app-restorecheck BAD=0 MODE=APP_LEVEL_RESTORE_CHECKS_TWO_CLOUDS FULL_VM_RESTORE_AVAILABLE=NO TOTAL_APPS=30 OK_APPS=30 FAIL_LIST=none REPORT=/var/lib/homelab-node-loss/pve01-core-monitoring-app-level/app-restorecheck-latest.md
1118:/var/lib/homelab-health/hapusya-site-final-seal.txt:1:STATUS=OK TS=2026-07-13T12:33:08Z TYPE=hapusya-site-final-seal BAD=0 APP=hapusya-site URL=https://hapusya.ru/ PRESEAL_DESIRED_STATE_HEAD=69966eb83fc065141df3c46df0d6c0f282275046 ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb APPLY_MANIFEST_SHA256=f44992e3e0cc443b372779cb35d9e899d904a9960a9de03c14559886fdb1fc75 PRODUCTION_BROWSER_MANIFEST_SHA256=9f3cdf3063f8724d1d1c1ad619f47d6f98d434c3422a74903585146858a8dfac BROWSER_SMOKE=OK BROWSER_RUNS=5 BROWSER_ERRORS=0 DESKTOP_VIDEO=OK MOBILE_VIDEO=OK STATIC_FALLBACK=OK BACKUP_JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 TWO_CLOUDS=OK RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK DESIRED_STATE_SYNC=OK GENERATOR_SHA256=507bbbc2691f1f08cb1dd3660ed76cf13a8c36c23833d5fdec40cf6e07486cb8 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 POLICY_SHA256=4bf996ff451ae3ec56eb2fcd42842606cd97295b4d5141aedb135c956395e4b5 EVIDENCE_ROOT=/var/lib/homelab-change-sets/hapusya-site-final-seal/20260713T123308Z EVIDENCE_MANIFEST_SHA256=7eb993a17ac18a1f7c9ae0b52d745d3778833d15626ed34c82496f74916eb26a PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY DESIRED_STATE_CHANGED=YES_FINAL_SEAL_COMMIT_ONLY ALL_HAPUSYA_SITE_TAILS_CLOSED=YES TASK_COMPLETE=YES
1120:/var/lib/homelab-health/pve03-failed-unit-cleanup-closure.txt:1:STATUS=OK TS=2026-07-07T12:52:21Z TYPE=pve03-failed-unit-cleanup-closure BAD=0 WARN=0 MODE=READ_ONLY_CLOSURE LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CP_MATCH=YES RESTORE_CHECK=YES PVE03_FAILED_UNITS=0 REPORT=/var/lib/homelab-pve03-failed-unit-cleanup/closure-latest.md
1128:/var/lib/homelab-health/cluster-admin-backup-webpanel-final-seal.txt:1:STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[PRIVATE_IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md
1132:/var/lib/homelab-health/appbackupctl-routing.txt:1:STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z
1134:/var/lib/homelab-health/cluster-admin-stage3-api-apply.txt:1:STATUS=OK TS=2026-07-13T03:22:35Z TYPE=cluster-admin-stage3-api-apply BAD=0 COMMAND_PHYSICAL_LINES=1 RESULT=APPLIED PRODUCTION_CHANGED=LIVE_APP_AND_DESIRED_STATE PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=YES CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_PREAPPLY_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES PREAPPLY_BACKUP_REUSED=YES AST_IDENTICAL_NORMALIZATION=YES EXISTING_ROUTES_BASELINE=OK NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES OPENAPI_DOCS_EXPOSURE_CHANGED=NO CORE_DATABASE_STATE=1|1|9|7|1 INCIDENT_ENGINE=ACTIVE VM180_FAILED_UNITS=0 ROLLBACK_CHECKPOINT=VERIFIED OLD_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a NEW_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 READY_FOR_STAGE3_AUTHENTICATED_PRODUCTION_VERIFY=YES STAGE3_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-production-apply/20260713T032235Z
1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL
1146:/var/lib/homelab-health/special-vm-emergency-non-app-final-seal.txt:1:STATUS=OK TS=2026-07-14T01:15:48Z TYPE=special-vm-emergency-non-app-final-seal BAD=0 WARN=0 FAMILY_COVERAGE=11/11 MATRIX_STATUS=OK JOBS_TOTAL=40 JOBS_OK=36 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 COMPATIBILITY_OUTPUTS=31 REQUIRED_GREEN_JOBS=14 VM_FULL=OK VM_LOCAL_DUMPS=OK EDGE_VM_SOURCE=OK EDGE_VM_CLOUD=OK NEXTCLOUD_VM_SOURCE=homelab_vm_local_dumps_2cloud NEXTCLOUD_VM_CLOUD=OK EMERGENCY_PACK=OK SKLADCHIK_REPORTS_MONITOR=OK GOLDEN_STATE=OK ROUTER_RUNNING_CONFIG=OK MAIL_CLOUD_CRITICAL=OK MAIL_CLOUD_EDGE=OK MAIL_CLOUD_NEXTCLOUD=OK WRITE_MODE=legacy READ_MODE=dual MIGRATE_EXISTING=no SOURCE_BACKUP_RUN=NO CLOUD_OPERATIONS_RUN=NO RETENTION_RUN=NO CLOUD_PURGE_RUN=NO CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=b26f498b55de5d80bcbb4b9fb3a9bccded9f99b4 CURRENT_SPECIAL_VM_EMERGENCY_NON_APP_STAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/special-vm-emergency-non-app-final-seal/20260714T011514Z
1151:/var/lib/homelab-health/crypto-portfolio-ui-preapply-backup.txt:1:STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z
1157:/var/lib/homelab-health/legacy-monitoring-backups-migrated.txt:1:STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK
1159:/var/lib/homelab-health/cluster-admin-stage2-final-seal.txt:1:STATUS=OK TS=2026-07-13T01:45:29Z TYPE=cluster-admin-stage2-final-seal BAD=0 STAGE1_CLOSED=YES STAGE2_APPLY=OK PRODUCTION_CHANGED=BACKUP_HEALTH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_SCHEMA_CHANGED=NO_ADDITIONAL_CHANGE_DURING_SEAL INCIDENT_ENGINE_RESTARTED=NO MIGRATION_ID=002_canonical_foundation_sync FUNCTION=cluster_admin_sync_foundation COMMITTED_SYNC=YES MULTICYCLE_COLLECTOR_TEST=OK MULTICYCLE_NEW_RUNS=2 MULTICYCLE_GOOD_RUNS=2 ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK RETIRED_PLACEHOLDERS_ALLOWED=YES FOUNDATION_COUNTS=85|21|2|229|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0|0 ORPHANS=0 POSTAPPLY_DIRECT_SNAPSHOT=VERIFIED POSTAPPLY_RESTORE_TEST=OK POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T014757Z LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE2_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage2/stage2-final-seal-latest.txt REPORT_SHA256=32d4d37d82aa25194d789083d0e7ad2e9052ee38a72df5312b711226898cf1a6 BLOCKERS=/var/lib/homelab-cluster-admin-stage2/20260713T014529Z/blockers.txt
1169:/var/lib/homelab-health/cluster-admin-stage3-final-seal.txt:1:STATUS=OK TS=2026-07-13T03:46:30Z TYPE=cluster-admin-stage3-final-seal BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=HEALTH_REFRESH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO STAGE2_CLOSED=YES STAGE3_APPLY=OK AUTHENTICATED_PRODUCTION_VERIFY=OK AUTHENTICATED_PRODUCTION_TESTS=25 APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 NEW_GET_ROUTES=7 DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY OPENAPI_DOCS_EXPOSURE=PRESERVED_DISABLED POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z POSTAPPLY_APPBACKUP_RESTORE_CHECK=OK COLLECTOR_STABILITY=OK LATEST_COLLECTOR_RUN_ID=3114 CANONICAL_CONSISTENCY=0|0|0|0|0 OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE3_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage3/stage3-final-seal-latest.txt REPORT_SHA256=6f67545cd7f0a75b87146c1df74c61c3b88e142f9e53a96b083ec27209e6d265
1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL
1172:/var/lib/homelab-health/backup-matrix-desired-state-persistence.txt:1:STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z
1175:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES
1184:/var/lib/homelab-health/cluster-admin-stage4d-adapter-candidates-in-isolation.txt:1:STATUS=OK TS=2026-07-14T06:14:58Z TYPE=cluster-admin-stage4d-adapter-candidates-in-isolation-final-seal BAD=0 STAGE4D_CLOSED=YES NEXT_STAGE=4E_RESTRICTED_PROBE_AGENT_DESIGN PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO APPLICATION_CHANGED=NO ENVELOPE_TOTAL=48 PROMETHEUS_ENVELOPES=5 BACKUP_MATRIX_ENVELOPES=40 HEALTH_ENVELOPES=3 LOCAL_REMOTE_OUTPUT_MATCH=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=e77fb37a0fa0033fe46558045e31dd92b6d1238a DESIRED_STATE_CLEAN=YES PVE01_FAILED_UNITS=0 REPORT=/var/lib/homelab-cluster-admin-stage4/20260714T061458Z/stage4d-adapter-candidates-in-isolation-seal.txt REPORT_SHA256=da6244c0a8a458b508fd58a0fec9cb1e6fa8da98a36f44d1368cbe947f4b5835
1185:/var/lib/homelab-health/backup-matrix-cluster-desired-state-sync.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
1187:/var/lib/homelab-health/cluster-admin-stage3-api-runtime-dry-run.txt:1:STATUS=OK TS=2026-07-13T02:53:34Z TYPE=cluster-admin-stage3-api-runtime-dry-run BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=TEMPORARY_ISOLATED_RUNTIME_AND_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO PRODUCTION_SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ISOLATED_BIND=127.0.0.1:18081 TEMPORARY_DATABASE_RESTORED=YES TEMPORARY_DATABASE_DROPPED=YES TEMPORARY_DATABASE_UNCHANGED=YES RUNTIME_TESTS=41 RUNTIME_FAILED_TESTS=0 EXISTING_ROUTES_BACKWARD_COMPATIBLE=YES NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES AUTHENTICATED_NEW_ROUTES=OK PAGINATION=OK FILTER_VALIDATION=OK ENTITY_DETAIL=OK ENTITY_DEPENDENCIES=OK SOURCES=OK POLICIES=OK COLLECTOR_RUNS=OK DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY FORBIDDEN_RESPONSE_FIELDS=ABSENT OPENAPI_DOCS_EXPOSURE_CHANGED=NO SECURITY_HEADERS=OK ISOLATED_RUNTIME_STOPPED=YES ISOLATED_PORT_CLOSED=YES LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK READY_FOR_STAGE3_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z RESULT=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z/runtime-result.json
1188:/var/lib/homelab-health/crypto-portfolio-remove-grt-failed-units-diagnostic.txt:1:STATUS=OK TS=2026-07-13T05:41:00Z TYPE=crypto-portfolio-remove-grt-failed-units-diagnostic BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=NO_DIAGNOSTIC_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO FAILED_STATE_RESET=NO CLOUD_WRITE_RUN=NO PVE01_FAILED_UNIT_COUNT=2 PVE01_FAILED_UNIT_NAMES=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service CRYPTO_PORTFOLIO_FAILED_UNIT_COUNT=0 OTHER_FAILED_UNIT_COUNT=2 CURRENT_CRYPTO_PRECHANGE_HEALTH=YES UNIT_DETAILS_CAPTURED=YES READY_FOR_TARGETED_FAILED_UNIT_DECISION=YES DIAGNOSTIC_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-failed-units-diagnostic/20260713T054100Z
1189:/var/lib/homelab-health/crypto-portfolio-remove-grt-production-apply.txt:1:STATUS=OK TS=2026-07-13T05:47:54Z TYPE=crypto-portfolio-remove-grt-production-apply BAD=0 COMMAND_PHYSICAL_LINES=1 APPLY_REVISION=V3 FIX_UID_READONLY=YES FIX_READINESS_ENDPOINT=YES FIX_ROLLBACK_DATABASE_METADATA=YES PRODUCTION_CHANGED=YES_REMOVE_GRT DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND PRODUCTION_CONTAINER_RECREATED=YES CLOUD_WRITE_RUN=NO ROLLBACK_ON_FAILURE=ARMED_NOT_NEEDED TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 PRECHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CHART_HEALTH_SCRIPT_SHA256=c16473f16390eeeff4d4eb701db604093564dd78d0be2d30b4cc8ffeeb22364a ACTIVE_COINS=HBAR,POL GRT_SOURCE_REFERENCES=0 GRT_DATABASE_ROWS=0|0|0 GRT_MARKET_ENTRY=ABSENT GRT_CACHE_ENTRIES=ABSENT DATABASE_MODE=0644 DATABASE_UID=10001 DATABASE_GID=10001 DATABASE_INTEGRITY=ok WORKER_STATUS=OK WORKER_COIN_COUNT=2 PUBLIC_CODES=200|303|401|200|200|401 CHART_HEALTH=OK CHART_SYMBOLS=HBAR,POL CHART_HEALTH_TIMER=ACTIVE VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_UNITS=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES REMOTE_STAGE_CLEANED=YES READY_FOR_GRT_POSTAPPLY_VERIFY_BACKUP_SYNC=YES APPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-production-apply-v3/20260713T054754Z
1193:/var/lib/homelab-health/crypto-portfolio-runtime-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES
1194:/var/lib/homelab-health/backup-matrix-cluster-preseal-convergence.txt:1:STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z
1202:/var/lib/homelab-health/node-loss-final-closure.txt:1:STATUS=OK TS=2026-07-08T16:35:56Z TYPE=node-loss-final-closure BAD=0 MODE=ALL_NODE_LOSS_DRILLS_SEALED PVE03_EDGE_RESTORE=OK PVE03_EDGE_FRESHNESS=OK PVE02_FORUM_RUNTIME=OK PVE01_NEXTCLOUD_DNS=OK PVE01_CORE_MONITORING_APPLEVEL=OK FULL_VM_GAP_VM170_VM171=ACKNOWLEDGED_APP_LEVEL_ONLY TEST_VMS_CLEANED=YES DNS_PAIR_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-node-loss/final-closure-latest.md
1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed
1332:/var/lib/homelab-health/codevipe-cloud-backup-reset.txt:1:STATUS=OK TS=2026-07-08T05:25:22Z TYPE=remote-failed-unit-codevipe-reset BAD=0 WARN=0 MODE=RESET_STALE_FAILED_STATE UNIT=codevipe-cloud-backup.service REASON=last_journal_success_and_no_timer ACTION=reset_failed_only
1426:● homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image
1499:2026-07-21T11:35:23Z	BACKUP-DUAL-CLOUD-PREFLIGHT-059	c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479	1	FAIL	backup-audit
1500:2026-07-21T12:28:26Z	BACKUP-VM-DUMP-PIPELINE-073	cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b	1	FAIL	backup-audit
1501:2026-07-21T12:42:53Z	BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075	a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936	143	FAIL	backup-audit
1502:2026-07-21T12:56:52Z	BACKUP-DRAIN-LOCAL-ARCHIVES-079	531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32	75	FAIL	backup-remediation
1503:2026-07-21T13:04:48Z	BACKUP-FINAL-PROOF-082	9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601	127	FAIL	backup-audit
1505:2026-07-21T13:14:27Z	CLUSTER-BACKUP-POSTCLEAN-PROOF-086	601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da	127	FAIL	backup-cleanup
1506:2026-07-21T13:15:21Z	BACKUP-POLICY-CLOSEOUT-087	d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52	1	FAIL	backup-cleanup
1507:2026-07-21T13:17:20Z	BACKUP-CLEANUP-FINAL-CLOSEOUT-088	5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6	1	FAIL	backup-cleanup
=== ACTIONABLE_P1 ===
=== ACTIONABLE_P1 ===
=== ACTIONABLE_P1 ===
26:1:# HOMELAB ASSISTANT ERROR REGISTER
28:77:21. Ошибка: nested PHP php -r дал Parse error на forum-prod.
29:83:Команда 108 дала PHP Parse error на smtpHost и пустой proof-файл.
30:88:Команда 111 не изменила smtpSsl: OLD_SMTPSSL=false, SQL ERROR 1064, NEW_SMTPSSL=false.
40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.
41:154:37. Coverage gap: pve03_staging missing from disk-space health coverage.
42:159:Факт: terminal output around command 222 showed paste artifact near "FAIL; fi".
45:171:Факт: first reconciliation wrote STATUS=ERROR because router-running-config-mail-cloud.txt is on pve01, not edge-vm.
51:194:- Rule: home portal closure requires current Homepage API-error logs to be zero after restart/reload, not only HTTP 200.
52:195:- If Homepage UI shows API error or logs contain current httpProxy/API timeout errors, the portal is REVIEW until the widget/API cause is disabled or fixed.
55:199:- Previous home portal cleanup command failed in local bash before execution: syntax error near unexpected token `('.
60:227:- Observed: CMD_TEXT=access-list ? returned Core::Configurator error execute denied [cli].
79:368:- Evidence: proof 542 showed all five zones missing and DNS record create probes failed.
83:388:- Impact: web routing is OK, but mail-related DNS may still contain stale provider data.
84:392:- Context: SPF cleanup dry-run command 563 had broken shell quoting and left the terminal at multiline prompt.
85:397:- Context: SPF cleanup command 565 attempted to replace stale SPF records.
93:416:- Cause: secret file with raw SMTP_PASS was later parsed/sourced by shell; shell error printed the raw assignment line.
95:429:- Action: remove forum-prod msmtp secret/config files so XenForo cannot keep retrying broken SMTP auth.
97:435:- Evidence: latest service run failed with missing /root/backups/codevipe; remote inventory contains old codevipe-public/codevipe.sql files, not current five-forum archive.
120:621:- Certbot on edge rejected --dns-cloudflare-* arguments: dns-cloudflare plugin missing/not loaded.
129:657:- Факт: bash начал интерпретировать Python-код, появились syntax error и вторичный prompt >.
131:675:- When asked to fix one missing Uptime Kuma service, add only one monitor, then generate proposals separately.
134:681:- Dockge inactive items after migration can be stale compose folders, not stopped containers.
135:684:- Do not delete containers to fix Dockge inactive UI; connect agents and archive only confirmed stale moved stack definitions.
145:737:- The six wrapper validation failures are cascading missing-marker checks and do not prove six independent infrastructure faults.
149:761:- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults.
153:774:- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects.
155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.
171:839:- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`.
180:865:- Symptom: error-register candidate construction stopped with RC1 before applying the candidate.
183:876:- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis.
201:948:BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative.
203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.
220:- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR.
235:- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring.
236:- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification.
245:- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first.
250:- Stale moved edge stack folders archived under /opt/dockge-stale-archive/20260702T230442Z on edge-vm; no runtime containers were deleted.
252:- Final target: remote not_visible count must be 0 and edge stale remaining matches must be 0.
258:- Stale moved stacks were archived, not deleted.
316:- Фактические поля collector_runs: finished_at и error_text; completed_at и error отсутствуют.
320:- Production осталась 0|0|OK, timer активен, failed units отсутствуют.
372:    "missing": [
459:STATUS=FAIL
475:JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000
486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
495:JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000
508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
510:JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000
513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
676:2026-07-08T09:26:36.7403618890 390 /var/lib/homelab-health/dockge-stale-stack-classification-after-agents.txt
681:2026-07-08T09:26:36.7406026680 390 /var/lib/homelab-health/homepage-missing-card-proposals.txt
684:2026-07-08T09:26:36.7408259230 390 /var/lib/homelab-health/homepage-missing-card-proposals-final.txt
689:2026-07-08T09:26:36.7410552890 389 /var/lib/homelab-health/dockge-stale-archive-last.txt
1114:/var/lib/homelab-health/service-registry.txt:1:STATUS=OK TS=2026-07-21T13:15:17Z TYPE=service-registry SERVICES=12 MISSING=0 REGISTRY=/var/lib/homelab-registry/service-registry-latest.tsv
1122:/var/lib/homelab-health/nextcloud-restore-proof-offhost.txt:1:STATUS=OK TS=20260624T190330Z TYPE=nextcloud-restore-proof-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge/nextcloud-aio/nextcloud-aio-restore-proof/20260624T120344Z ITEMS=1 OK=1 ERROR=0 BYTES=2135684 FILES=7 SECRET_PRINTED=NO
1127:/var/lib/homelab-health/crypto-portfolio-remove-grt-recovery-audit.txt:1:STATUS=ERROR TS=2026-07-13T05:39:07Z TYPE=crypto-portfolio-remove-grt-recovery-audit BAD=1 COMMAND_PHYSICAL_LINES=1 ROOT_CAUSE=READINESS_ENDPOINT_MISMATCH RECOVERY_VERIFIED=NO TASK_COMPLETE=NO READY_FOR_GRT_APPLY_SCRIPT_FIX=NO AUDIT_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-recovery-audit-v2/20260713T053907Z
1133:/var/lib/homelab-health/final-readiness.txt:1:STATUS=WARN TS=2026-07-21T05:31:30Z TYPE=final-readiness BAD=5 PASS=48 FAIL=5 REPORT=/var/lib/homelab-final-readiness/latest.md
1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL
1152:/var/lib/homelab-health/router-hard-power.txt:1:STATUS=ERROR TS=2026-07-16T06:20:34Z MODE=cycle REMOTE=[PRIVATE_IP] RC=74
1162:/var/lib/homelab-health/private-vpn-hosts.txt.bak-ok-wrapper-20260624T132705Z:16:STATUS=ERROR
1164:/var/lib/homelab-health/paperless-restore.txt:1:STATUS=STALE
1165:/var/lib/homelab-health/paperless-restore.txt:20:FRESHNESS=STALE
1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL
1174:/var/lib/homelab-health/ungated-health-backlog.txt:1:STATUS=OK TS=2026-07-07T07:45:01Z TYPE=ungated-health-backlog BAD=0 WARN=1 MODE=POST_PVE03_ROOT_CLEANUP_TRIAGE TOTAL=8 SUPERSEDED=4 STALE=2 POLICY_GAP_NEXT_STAGE=1 ACTIONABLE_CAPACITY=1 PVE03_ROOT_USAGE_PERCENT=31 PVE03_STAGING_USAGE_PERCENT=83 REPORT=/var/lib/homelab-ungated-health-backlog/latest.md
1190:/var/lib/homelab-health/mail-cloud-restore-drill.txt:1:STATUS=STALE
1191:/var/lib/homelab-health/mail-cloud-restore-drill.txt:20:FRESHNESS=STALE
1192:/var/lib/homelab-health/edge-restore-proofs-offhost.txt:1:STATUS=OK TS=20260624T190040Z TYPE=edge-restore-proofs-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge ITEMS=14 OK=14 ERROR=0 BYTES=226024287 SECRET_PRINTED=NO
1195:/var/lib/homelab-health/uptime-kuma-pending-health-monitors-fix.txt:8:NOTE=Paperless Restore may become red if its health file contains STATUS=ERROR; that is a real restore-health issue, not a Kuma pending issue.
1199:/var/lib/homelab-health/cluster-admin-stage0-convergence.txt:1:STATUS=WARN TS=2026-07-12T23:12:59Z TYPE=cluster-admin-stage0-targeted-convergence BAD=2 PROFILE_OK=NO CLUSTER_PASSPORT=FAIL FINAL_READINESS=FAIL OVERALL=FAIL DUTY_ADMIN=FAIL FAILED_UNITS=0 LIVE_MATCH_DESIRED_STATE=YES READY_FOR_FINAL_SEAL=NO REPORT_DIR=/var/lib/homelab-cluster-admin-stage0/readiness-profile-20260712T231259Z PRODUCTION_CHANGED=HEALTH_REGENERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO
1207:/var/lib/homelab-health/filebrowser-restore.txt:1:STATUS=STALE
1208:/var/lib/homelab-health/filebrowser-restore.txt:20:FRESHNESS=STALE
1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed
1494:2026-07-21T08:03:07Z	GITEA-PUSH-AUTH-001	fcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94	1	FAIL	gitea-runtime
1495:2026-07-21T08:13:32Z	GITEA-TOKEN-CLI-HELP-003	f1d0f967eed6490ad822944a42108c82dc4b59c9fd2557d8c64624dcb6853cd4	1	FAIL	gitea-runtime
1496:2026-07-21T08:14:52Z	GITEA-CLI-CAPABILITIES-004	2cd6c4d28c8f646d8063105fca80b2f80d38c6a5db3a3eb76869de326b12d2d6	1	FAIL	gitea-runtime
1497:2026-07-21T09:49:51Z	RUNTIME-RECEIVER-STATE-027	0831002c78c26f35826cd7199995e335ceadcd39f08b9bac23882a0fbb59c8cc	127	FAIL	gitea-runtime
1498:2026-07-21T10:26:18Z	RUNTIME-RECEIVER-CLEANUP-036	65fbcfbce38a55723a92a8790429d6567389321f030f02cb4bf301cbe67e2531	1	FAIL	gitea-runtime
1499:2026-07-21T11:35:23Z	BACKUP-DUAL-CLOUD-PREFLIGHT-059	c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479	1	FAIL	backup-audit
1500:2026-07-21T12:28:26Z	BACKUP-VM-DUMP-PIPELINE-073	cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b	1	FAIL	backup-audit
1501:2026-07-21T12:42:53Z	BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075	a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936	143	FAIL	backup-audit
1502:2026-07-21T12:56:52Z	BACKUP-DRAIN-LOCAL-ARCHIVES-079	531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32	75	FAIL	backup-remediation
1503:2026-07-21T13:04:48Z	BACKUP-FINAL-PROOF-082	9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601	127	FAIL	backup-audit
1504:2026-07-21T13:07:09Z	CLUSTER-CLEANUP-INVENTORY-083	09d3f14ca0205936a910dc7af7991474030273a47d76fa033ea6a3185c083922	1	FAIL	cluster-cleanup
1505:2026-07-21T13:14:27Z	CLUSTER-BACKUP-POSTCLEAN-PROOF-086	601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da	127	FAIL	backup-cleanup
1506:2026-07-21T13:15:21Z	BACKUP-POLICY-CLOSEOUT-087	d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52	1	FAIL	backup-cleanup
1507:2026-07-21T13:17:20Z	BACKUP-CLEANUP-FINAL-CLOSEOUT-088	5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6	1	FAIL	backup-cleanup
=== ACTIONABLE_STORAGE ===
=== ACTIONABLE_STORAGE ===
=== ACTIONABLE_STORAGE ===
40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.
454:local             dir     active       102626232        41916788        55450180   40.84%
=== OPEN_ERROR_REGISTER ===
=== FIRST_PRIORITY_CANDIDATES ===
38-BOOTSTRAP_SECTION_LINES=0
39-=== ACTIONABLE_P0 ===
40:86:398:- Issue: backup write permission problem caused record detection failure, so new v=spf1 -all records were created while old v=spf1 ip4:87.236.18.45 records remained.
41:155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.
42:203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.
43:486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
44:508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
45:513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
46:546:● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state
47:548:● homelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03
48:1102:/var/lib/homelab-health/backup-matrix-cluster-final-seal.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-final-seal BAD=0 CURRENT_STAGE_COMPLETE=YES MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 READ_WRITE_OVERLAP=0 VM_BACKUP_HEALTH=OK VM_FULL_BACKUP=OK EDGE_VM_CANONICAL_BACKUP=homelab-edge-vm-offhost-freshness.timer OLD_EDGE_TIMER_REENABLED=NO CLUSTER_PASSPORT=OK FINAL_READINESS=OK FINAL_READINESS_PASS=53 FINAL_READINESS_FAIL=0 OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 CLUSTER_NODES=3 QUORATE=YES GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 EVIDENCE_MANIFEST_SHA256=b4c68312026e1328b208a74090891f4809ab31ec69069e94a3d6902429bf3566 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO NEXT_ACTION=AUDIT_AND_MIGRATE_SPECIAL_VM_EMERGENCY_AND_NON_APP_BACKUP_FAMILIES EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
49:1104:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-production-closure.txt:1:STATUS=OK TS=2026-07-13T08:12:14Z TYPE=crypto-portfolio-ui-center-alert-currency-production-closure BAD=0 PRODUCTION_CHANGED=YES_UI_ONLY INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 POSTCHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION_ROLLBACK_COUNT=3 RETENTION_OPTIONAL_OTHER_CANDIDATES=1 RETENTION_REMOVED_ROLLBACK_COUNT=0 DANGLING_IMAGES_REMAINING=2 UNREFERENCED_DANGLING_IMAGES=0 DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE_CLEAN=YES PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES READY_FOR_UI_FINAL_SEAL=YES
50:1106:/var/lib/homelab-health/cluster-admin-stage3-postapply-backup.txt:1:STATUS=OK TS=2026-07-13T03:43:59Z TYPE=cluster-admin-stage3-postapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=POSTAPPLY_BACKUP_AND_CONTROLLED_COLLECTOR_RUN PRODUCTION_DATABASE_CHANGED=COLLECTOR_NORMAL_OPERATION_ONLY PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=YES_CONTROLLED_POSTAPPLY_BACKUP LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 APPBACKUP=VERIFIED APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z APPBACKUP_SIZE_BYTES=1791603 APPBACKUP_RESTORE_CHECK=OK APPBACKUP_ROUTING=VERIFIED COLLECTOR_CONTROL_RUN=OK COLLECTOR_RUN_ID=3113 CANONICAL_CONSISTENCY=0|0|0|0|0 INCIDENT_ENGINE=ACTIVE COLLECTOR_TIMER=ACTIVE VM180_FAILED_UNITS=0 READY_FOR_STAGE3_FINAL_SEAL=YES STAGE3_CLOSED=NO POSTAPPLY_ROOT=/var/lib/homelab-stage3-postapply/20260713T034359Z
51:1109:/var/lib/homelab-health/pve01-core-monitoring-app-restorecheck.txt:1:STATUS=OK TS=2026-07-08T15:57:27Z TYPE=pve01-core-monitoring-app-restorecheck BAD=0 MODE=APP_LEVEL_RESTORE_CHECKS_TWO_CLOUDS FULL_VM_RESTORE_AVAILABLE=NO TOTAL_APPS=30 OK_APPS=30 FAIL_LIST=none REPORT=/var/lib/homelab-node-loss/pve01-core-monitoring-app-level/app-restorecheck-latest.md
52:1118:/var/lib/homelab-health/hapusya-site-final-seal.txt:1:STATUS=OK TS=2026-07-13T12:33:08Z TYPE=hapusya-site-final-seal BAD=0 APP=hapusya-site URL=https://hapusya.ru/ PRESEAL_DESIRED_STATE_HEAD=69966eb83fc065141df3c46df0d6c0f282275046 ROUTE_SHA256=02d65cb0082a04cae282126cc829a002f171e863291c18ba04d788dc2d6282ca UNIT_SHA256=b3b07137b25ac0a0b4bdadadad654cf209387ade86a2ef2cdca58e8a7c0cc212 RELATIVE_TREE_SHA256=9dadf21da041bc4d6ee4a42f5b3ac7719c16bcc39052de32af895c6cd28286cb APPLY_MANIFEST_SHA256=f44992e3e0cc443b372779cb35d9e899d904a9960a9de03c14559886fdb1fc75 PRODUCTION_BROWSER_MANIFEST_SHA256=9f3cdf3063f8724d1d1c1ad619f47d6f98d434c3422a74903585146858a8dfac BROWSER_SMOKE=OK BROWSER_RUNS=5 BROWSER_ERRORS=0 DESKTOP_VIDEO=OK MOBILE_VIDEO=OK STATIC_FALLBACK=OK BACKUP_JOB=homelab-pve01-app-hapusya-site-20260713T115259Z ARCHIVE_SHA256=bf284292974600384c43c47fc99be6fedf0ebd68f722b402a7260f1df680d167 TWO_CLOUDS=OK RESTORE_CHECK=OK_BOTH_CLOUDS RETENTION=OK DESIRED_STATE_SYNC=OK GENERATOR_SHA256=507bbbc2691f1f08cb1dd3660ed76cf13a8c36c23833d5fdec40cf6e07486cb8 MAP_SHA256=815b52dcedf7e3e92e07939bade32c7e8ed6acc0f9e064b65db6daec9f559c63 POLICY_SHA256=4bf996ff451ae3ec56eb2fcd42842606cd97295b4d5141aedb135c956395e4b5 EVIDENCE_ROOT=/var/lib/homelab-change-sets/hapusya-site-final-seal/20260713T123308Z EVIDENCE_MANIFEST_SHA256=7eb993a17ac18a1f7c9ae0b52d745d3778833d15626ed34c82496f74916eb26a PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY DESIRED_STATE_CHANGED=YES_FINAL_SEAL_COMMIT_ONLY ALL_HAPUSYA_SITE_TAILS_CLOSED=YES TASK_COMPLETE=YES
53:1120:/var/lib/homelab-health/pve03-failed-unit-cleanup-closure.txt:1:STATUS=OK TS=2026-07-07T12:52:21Z TYPE=pve03-failed-unit-cleanup-closure BAD=0 WARN=0 MODE=READ_ONLY_CLOSURE LATEST_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CONTROL_PLANE=homelab-pve01-app-safety-net-control-plane-20260707T124031Z GOLDEN_CP_MATCH=YES RESTORE_CHECK=YES PVE03_FAILED_UNITS=0 REPORT=/var/lib/homelab-pve03-failed-unit-cleanup/closure-latest.md
54:1128:/var/lib/homelab-health/cluster-admin-backup-webpanel-final-seal.txt:1:STATUS=OK TS=2026-07-09T05:24:54Z TYPE=cluster-admin-backup-webpanel-final-seal BAD=0 VMID=180 URL=http://[PRIVATE_IP]:8080/ BACKUP=MAIL_CLOUD_2COPY_VERIFY_OK WEBPANEL=OK WEBPANEL_SYNC=OK OVERALL_HARDGATE=YES FINAL_READINESS_HARDGATE=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-ideal-cluster/cluster-admin-backup-webpanel-final-seal-latest.md
55:1132:/var/lib/homelab-health/appbackupctl-routing.txt:1:STATUS=OK TS=2026-07-13T00:29:36Z TYPE=appbackupctl-routing BAD=0 RESULT=APPLIED PRODUCTION_CHANGED=BACKUP_CLI_ROUTING_AND_DESIRED_STATE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO DEFAULT_ROUTE=dockge-stacks CLUSTER_ADMIN_ROUTE=homelab-cluster-admin-incident-engine-backup.service UNSUPPORTED_TARGET_POLICY=REFUSE_RC_2 ROUTE_TEST_DOCKGE=OK ROUTE_TEST_CLUSTER_ADMIN=OK ROUTE_TEST_DEFAULT=OK ROUTE_TEST_UNSUPPORTED=OK FRAMEWORK_CHECK=OK UNSUPPORTED_ROUTE_BACKUP_CREATED=NO LIVE_MATCH_DESIRED_STATE=YES PVE01_FAILED_UNITS=0 OVERALL=OK DUTY_ADMIN=OK STAGE1_CLOSED=YES STAGE2_READY=YES OLD_HEAD=bd5eb856a8d61a5fb83c2bcaf74cb8ac98f075c5 NEW_HEAD=4d0d03395d2baf4024ad7c9d5870f21bdaee9f99 CHANGE_SET=/var/lib/homelab-change-sets/appbackupctl-routing/20260713T002936Z
56:1134:/var/lib/homelab-health/cluster-admin-stage3-api-apply.txt:1:STATUS=OK TS=2026-07-13T03:22:35Z TYPE=cluster-admin-stage3-api-apply BAD=0 COMMAND_PHYSICAL_LINES=1 RESULT=APPLIED PRODUCTION_CHANGED=LIVE_APP_AND_DESIRED_STATE PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO SERVICE_RESTARTED=YES CLOUD_WRITE_RUN=NO_REUSE_VERIFIED_PREAPPLY_BACKUP APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES PREAPPLY_BACKUP_REUSED=YES AST_IDENTICAL_NORMALIZATION=YES EXISTING_ROUTES_BASELINE=OK NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES OPENAPI_DOCS_EXPOSURE_CHANGED=NO CORE_DATABASE_STATE=1|1|9|7|1 INCIDENT_ENGINE=ACTIVE VM180_FAILED_UNITS=0 ROLLBACK_CHECKPOINT=VERIFIED OLD_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a NEW_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 READY_FOR_STAGE3_AUTHENTICATED_PRODUCTION_VERIFY=YES STAGE3_CLOSED=NO CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-production-apply/20260713T032235Z
57:1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL
58:1146:/var/lib/homelab-health/special-vm-emergency-non-app-final-seal.txt:1:STATUS=OK TS=2026-07-14T01:15:48Z TYPE=special-vm-emergency-non-app-final-seal BAD=0 WARN=0 FAMILY_COVERAGE=11/11 MATRIX_STATUS=OK JOBS_TOTAL=40 JOBS_OK=36 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 COMPATIBILITY_OUTPUTS=31 REQUIRED_GREEN_JOBS=14 VM_FULL=OK VM_LOCAL_DUMPS=OK EDGE_VM_SOURCE=OK EDGE_VM_CLOUD=OK NEXTCLOUD_VM_SOURCE=homelab_vm_local_dumps_2cloud NEXTCLOUD_VM_CLOUD=OK EMERGENCY_PACK=OK SKLADCHIK_REPORTS_MONITOR=OK GOLDEN_STATE=OK ROUTER_RUNNING_CONFIG=OK MAIL_CLOUD_CRITICAL=OK MAIL_CLOUD_EDGE=OK MAIL_CLOUD_NEXTCLOUD=OK WRITE_MODE=legacy READ_MODE=dual MIGRATE_EXISTING=no SOURCE_BACKUP_RUN=NO CLOUD_OPERATIONS_RUN=NO RETENTION_RUN=NO CLOUD_PURGE_RUN=NO CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 CLUSTER_PASSPORT=OK DUTY_ADMIN=OK OVERALL=OK OVERALL_BAD=0 FINAL_READINESS=OK FINAL_PASS=53 FINAL_FAIL=0 WEBPANEL=OK PUBLIC_OVERALL=OK FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DESIRED_STATE_PRE_SEAL_HEAD=b26f498b55de5d80bcbb4b9fb3a9bccded9f99b4 CURRENT_SPECIAL_VM_EMERGENCY_NON_APP_STAGE_COMPLETE=YES EVIDENCE_ROOT=/var/lib/homelab-change-sets/special-vm-emergency-non-app-final-seal/20260714T011514Z
59:1151:/var/lib/homelab-health/crypto-portfolio-ui-preapply-backup.txt:1:STATUS=OK TS=2026-07-13T07:42:53Z TYPE=crypto-portfolio-ui-preapply-backup BAD=0 COMMAND_PHYSICAL_LINES=1 PREVIOUS_FAILURE=CORRECTED PREVIOUS_FAILURE_ROOT_CAUSE=INLINE_PYTHON_QUOTE_ERROR_IN_NONESSENTIAL_HELPER_PROOF BACKUP_SERVICE_RERUN=NO MANUAL_REHYDRATE_RERUN=NO CANONICAL_RESTORE_CHECK_RERUN=YES_READ_ONLY PRODUCTION_CHANGED=NO PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CONTAINER_RECREATED=NO CLOUD_WRITE_RUN=NO_READ_ONLY_RESTORE_CHECK TARGET_HOST=core-apps.gram1.ru INPUT_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CANDIDATE_INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T071127Z BACKUP_STATUS=success BACKUP_SCHEMA=homelab-app-backup-manifest-v2-flat BACKUP_LAYOUT=flat-v1 BACKUP_PATH_REL=hb/a/cp-260713-071127 BACKUP_DATABASE_INTEGRITY=ok BACKUP_FILE_COUNT=19 BACKUP_ARCHIVE_SIZE_BYTES=136632 BACKUP_ARCHIVE_SHA256=b29eb55820ec46c474f51e995e92e371251b6df717ac98182d807d1dbf8473f5 BACKUP_LOCAL_STAGING_DELETED=YES CANONICAL_TWO_CLOUD_RESTORE_CHECK=OK APPBACKUPCTL_RESTORE_CHECK=OK_BOTH_CLOUDS ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-ui-center-alert-currency-prechange-20260713T071125Z ROLLBACK_MANIFEST_SHA256=0ef56c8f021f549777131ffe878cd5a263bdfb6f1dfb8bb080cefdd554573aca ROLLBACK_INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e PRODUCTION_UNCHANGED=YES PRODUCTION_HEALTH=200 VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES CHART_HEALTH_TIMER=ACTIVE BACKUP_TIMER=ACTIVE DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 DESIRED_STATE_CLEAN=YES DESIRED_STATE_REMOTE_COUNT=0 READY_FOR_UI_PRODUCTION_APPLY=YES PREAPPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-ui-preapply-backup-canonical-close/20260713T074253Z
60:1157:/var/lib/homelab-health/legacy-monitoring-backups-migrated.txt:1:STATUS=OK TS=2026-07-10T20:00:03Z TYPE=legacy-monitoring-backups-migrated BAD=0 APPS=6 RESTORE_FAILURES=0 ACTIVE_LEGACY_STATES=0 OFFHOST=OK RESTORE=OK
61:1159:/var/lib/homelab-health/cluster-admin-stage2-final-seal.txt:1:STATUS=OK TS=2026-07-13T01:45:29Z TYPE=cluster-admin-stage2-final-seal BAD=0 STAGE1_CLOSED=YES STAGE2_APPLY=OK PRODUCTION_CHANGED=BACKUP_HEALTH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_SCHEMA_CHANGED=NO_ADDITIONAL_CHANGE_DURING_SEAL INCIDENT_ENGINE_RESTARTED=NO MIGRATION_ID=002_canonical_foundation_sync FUNCTION=cluster_admin_sync_foundation COMMITTED_SYNC=YES MULTICYCLE_COLLECTOR_TEST=OK MULTICYCLE_NEW_RUNS=2 MULTICYCLE_GOOD_RUNS=2 ACTIVE_CURRENT_SET_CONVERGENCE=OK NATURAL_KEY_CONVERGENCE=OK EPHEMERAL_ID_REFRESH=OK RETIRED_PLACEHOLDERS_ALLOWED=YES FOUNDATION_COUNTS=85|21|2|229|26|0 DYNAMIC_INVARIANTS=0|0|21|21|0|0|0|0|0|0|0 ORPHANS=0 POSTAPPLY_DIRECT_SNAPSHOT=VERIFIED POSTAPPLY_RESTORE_TEST=OK POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T014757Z LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE2_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage2/stage2-final-seal-latest.txt REPORT_SHA256=32d4d37d82aa25194d789083d0e7ad2e9052ee38a72df5312b711226898cf1a6 BLOCKERS=/var/lib/homelab-cluster-admin-stage2/20260713T014529Z/blockers.txt
62:1169:/var/lib/homelab-health/cluster-admin-stage3-final-seal.txt:1:STATUS=OK TS=2026-07-13T03:46:30Z TYPE=cluster-admin-stage3-final-seal BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=HEALTH_REFRESH_AND_SEAL_ARTIFACTS_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO STAGE2_CLOSED=YES STAGE3_APPLY=OK AUTHENTICATED_PRODUCTION_VERIFY=OK AUTHENTICATED_PRODUCTION_TESTS=25 APP_SHA256=a330a2d33ee5a8a57778e3541179e3faebcc0531600a504df744435e80a93b34 LIVE_MATCH_DESIRED_STATE=YES DESIRED_STATE_HEAD=b974057b536ece861299d0b230c06cc0e6702c39 NEW_GET_ROUTES=7 DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY OPENAPI_DOCS_EXPOSURE=PRESERVED_DISABLED POSTAPPLY_APPBACKUP=VERIFIED POSTAPPLY_APPBACKUP_JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T034400Z POSTAPPLY_APPBACKUP_RESTORE_CHECK=OK COLLECTOR_STABILITY=OK LATEST_COLLECTOR_RUN_ID=3114 CANONICAL_CONSISTENCY=0|0|0|0|0 OBSERVER=OK FULL_OBSERVER=OK RESTRICTED_PROBES=OK FINAL_READINESS=OK CLUSTER_PASSPORT=OK OVERALL=OK DUTY_ADMIN=OK VM180_FAILED_UNITS=0 PVE01_FAILED_UNITS=0 CURRENT_HEALTH_RECHECKED=YES STAGE3_CLOSED=YES REPORT=/var/lib/homelab-cluster-admin-stage3/stage3-final-seal-latest.txt REPORT_SHA256=6f67545cd7f0a75b87146c1df74c61c3b88e142f9e53a96b083ec27209e6d265
63:1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL
64:1172:/var/lib/homelab-health/backup-matrix-desired-state-persistence.txt:1:STATUS=OK TS=2026-07-13T17:40:37Z TYPE=backup-matrix-desired-state-persistence BAD=0 GENERATED_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_PARENT=25568fab24c0594418b3676dc511c8234b4e1ae5 GENERATOR_SHA256=19da7f049b4f7f7286faf292c5419c37cc1e1768c2d8fe36b257ab0cf9c18874 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 DESIRED_MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATUS=OK JOBS_BAD=0 FAILED_UNITS=0 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO ROLLBACK=/var/lib/homelab-incidents/backup-matrix-desired-state-persistence-20260713T174036Z
65:1175:/var/lib/homelab-health/crypto-portfolio-ui-center-alert-currency-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES
66:1184:/var/lib/homelab-health/cluster-admin-stage4d-adapter-candidates-in-isolation.txt:1:STATUS=OK TS=2026-07-14T06:14:58Z TYPE=cluster-admin-stage4d-adapter-candidates-in-isolation-final-seal BAD=0 STAGE4D_CLOSED=YES NEXT_STAGE=4E_RESTRICTED_PROBE_AGENT_DESIGN PRODUCTION_CHANGED=NO DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO APPLICATION_CHANGED=NO ENVELOPE_TOTAL=48 PROMETHEUS_ENVELOPES=5 BACKUP_MATRIX_ENVELOPES=40 HEALTH_ENVELOPES=3 LOCAL_REMOTE_OUTPUT_MATCH=OK PRODUCTION_STATE=0|0|OK DESIRED_STATE_HEAD=e77fb37a0fa0033fe46558045e31dd92b6d1238a DESIRED_STATE_CLEAN=YES PVE01_FAILED_UNITS=0 REPORT=/var/lib/homelab-cluster-admin-stage4/20260714T061458Z/stage4d-adapter-candidates-in-isolation-seal.txt REPORT_SHA256=da6244c0a8a458b508fd58a0fec9cb1e6fa8da98a36f44d1368cbe947f4b5835
67:1185:/var/lib/homelab-health/backup-matrix-cluster-desired-state-sync.txt:1:STATUS=OK TS=2026-07-13T17:51:31Z TYPE=backup-matrix-cluster-desired-state-sync BAD=0 PRE_SYNC_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 GENERATED_SYNC_HEAD=597119391e7b265b85fd9a76644e4e33af47e469 GENERATED_SYNC_MODE=NEW_GENERATED_COMMIT GENERATED_SYNC_COMMIT_DELTA=1 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_OK=31 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 FAILED_UNITS=0 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=14a31112aac5d577ae01df567921b093b2fedcf7189c4cc8f009216ee0749380 PRESEAL_SHA256=7ea1f710cd311bd51dba05a313b4c540f57d97407f008881e502fc213b4b3c99 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-final-seal/20260713T175131Z
68:1187:/var/lib/homelab-health/cluster-admin-stage3-api-runtime-dry-run.txt:1:STATUS=OK TS=2026-07-13T02:53:34Z TYPE=cluster-admin-stage3-api-runtime-dry-run BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=TEMPORARY_ISOLATED_RUNTIME_AND_EVIDENCE_ONLY PRODUCTION_DATABASE_CHANGED=NO PRODUCTION_SCHEMA_CHANGED=NO LIVE_APPLICATION_CODE_CHANGED=NO DESIRED_STATE_CHANGED=NO PRODUCTION_SERVICE_RESTARTED=NO CLOUD_WRITE_RUN=NO APP_PRECHANGE_SHA256=c2748b8f96df842f491b5d2790790594222549853d0e2336cf60ecd6d4de0020 APP_CANDIDATE_SHA256=672d6a2975dd0e093788e7eb81a76e7f901ea6374ca4092a25ba0091aba1cb50 ISOLATED_BIND=127.0.0.1:18081 TEMPORARY_DATABASE_RESTORED=YES TEMPORARY_DATABASE_DROPPED=YES TEMPORARY_DATABASE_UNCHANGED=YES RUNTIME_TESTS=41 RUNTIME_FAILED_TESTS=0 EXISTING_ROUTES_BACKWARD_COMPATIBLE=YES NEW_GET_ROUTES=7 UNAUTHENTICATED_NEW_ROUTES_401=YES AUTHENTICATED_NEW_ROUTES=OK PAGINATION=OK FILTER_VALIDATION=OK ENTITY_DETAIL=OK ENTITY_DEPENDENCIES=OK SOURCES=OK POLICIES=OK COLLECTOR_RUNS=OK DATABASE_TRANSACTION_MODE=BEGIN_READ_ONLY FORBIDDEN_RESPONSE_FIELDS=ABSENT OPENAPI_DOCS_EXPOSURE_CHANGED=NO SECURITY_HEADERS=OK ISOLATED_RUNTIME_STOPPED=YES ISOLATED_PORT_CLOSED=YES LIVE_APP_UNCHANGED=YES PRODUCTION_RUNTIME_HEALTH=OK READY_FOR_STAGE3_PREAPPLY_BACKUP=YES DESIRED_STATE_HEAD=8f888e5f2063e49da918b31584c9ea4bcf53af3a CHANGE_SET=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z RESULT=/var/lib/homelab-change-sets/stage3-api-runtime-dry-run/20260713T025334Z/runtime-result.json
69:1188:/var/lib/homelab-health/crypto-portfolio-remove-grt-failed-units-diagnostic.txt:1:STATUS=OK TS=2026-07-13T05:41:00Z TYPE=crypto-portfolio-remove-grt-failed-units-diagnostic BAD=0 COMMAND_PHYSICAL_LINES=1 PRODUCTION_CHANGED=NO_DIAGNOSTIC_ONLY PRODUCTION_DATABASE_CHANGED=NO DESIRED_STATE_CHANGED=NO SERVICE_RESTARTED=NO FAILED_STATE_RESET=NO CLOUD_WRITE_RUN=NO PVE01_FAILED_UNIT_COUNT=2 PVE01_FAILED_UNIT_NAMES=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service CRYPTO_PORTFOLIO_FAILED_UNIT_COUNT=0 OTHER_FAILED_UNIT_COUNT=2 CURRENT_CRYPTO_PRECHANGE_HEALTH=YES UNIT_DETAILS_CAPTURED=YES READY_FOR_TARGETED_FAILED_UNIT_DECISION=YES DIAGNOSTIC_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-failed-units-diagnostic/20260713T054100Z
70:1189:/var/lib/homelab-health/crypto-portfolio-remove-grt-production-apply.txt:1:STATUS=OK TS=2026-07-13T05:47:54Z TYPE=crypto-portfolio-remove-grt-production-apply BAD=0 COMMAND_PHYSICAL_LINES=1 APPLY_REVISION=V3 FIX_UID_READONLY=YES FIX_READINESS_ENDPOINT=YES FIX_ROLLBACK_DATABASE_METADATA=YES PRODUCTION_CHANGED=YES_REMOVE_GRT DESIRED_STATE_CHANGED=NO_BY_THIS_COMMAND PRODUCTION_CONTAINER_RECREATED=YES CLOUD_WRITE_RUN=NO ROLLBACK_ON_FAILURE=ARMED_NOT_NEEDED TARGET_HOST=core-apps.gram1.ru DESIRED_STATE_HEAD=80763a6c62b94cde4f3293f3b529980f76919d25 PRECHANGE_BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T051015Z ROLLBACK_PATH=/var/lib/homelab-incidents/crypto-remove-grt-prechange-20260713T051013Z APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=8bbe88f5f420fe429d479c598e1f3f904521eb166d4707e32e579851e84c205e CHART_HEALTH_SCRIPT_SHA256=c16473f16390eeeff4d4eb701db604093564dd78d0be2d30b4cc8ffeeb22364a ACTIVE_COINS=HBAR,POL GRT_SOURCE_REFERENCES=0 GRT_DATABASE_ROWS=0|0|0 GRT_MARKET_ENTRY=ABSENT GRT_CACHE_ENTRIES=ABSENT DATABASE_MODE=0644 DATABASE_UID=10001 DATABASE_GID=10001 DATABASE_INTEGRITY=ok WORKER_STATUS=OK WORKER_COIN_COUNT=2 PUBLIC_CODES=200|303|401|200|200|401 CHART_HEALTH=OK CHART_SYMBOLS=HBAR,POL CHART_HEALTH_TIMER=ACTIVE VM170_FAILED_UNITS=0 PVE01_UNRELATED_FAILED_UNITS=homelab-backup-matrix-refresh.service,homelab-cluster-admin-pull.service PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES REMOTE_STAGE_CLEANED=YES READY_FOR_GRT_POSTAPPLY_VERIFY_BACKUP_SYNC=YES APPLY_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-production-apply-v3/20260713T054754Z
71:1193:/var/lib/homelab-health/crypto-portfolio-runtime-final-seal.txt:1:STATUS=OK TS=2026-07-13T08:37:54Z TYPE=crypto-portfolio-ui-center-alert-currency-final-seal BAD=0 DANGLING_IMAGE_REFERENCE_GATE=OK DESIRED_STATE_SYNC_ARTIFACT_GATE=OK APP=crypto-portfolio PRODUCTION_CHANGED=NO_FINAL_SEAL_ONLY APP_SHA256=44df2b25183b7c5d87ff1516ec2d9cd1d169acde3031f7c2aa1fc4e0c7813cd2 WORKER_SHA256=174a14b86014535980d200a76d0057110c7af102c66f16c2e21fc3e74bb2e49e INDEX_SHA256=e497a36d8549654c6d451360a11a2ad714e19c23e6d384330a5436f7d32267cb LIBRARY_SHA256=c0992580867c4912cc9385b3c2728315bcc1a76c7f1087dca908430fccdf31d7 ACTIVE_COINS=HBAR,POL GRT_REMOVED=YES GRT_ROWS=0|0|0 GRT_SOURCE_REFERENCES=0 GRT_CACHE_ENTRIES=0 SHARP_MOVES_CENTERED=YES ALERT_CURRENCY_USD_RUB=OK BROWSER_SMOKE=OK BROWSER_RUNS=2 BROWSER_ERRORS=0 BROWSER_RESULT_SHA256=b566f492cd80c8f99bc6b24cbef6d5f326c04f4d7168359d05eb1f189589e733 SCREENSHOT_SHA256=00b2853bee9c880146d0dd75fac60feb7e46d4c129ee06754047ef30d90bd4ba BACKUP_JOB=homelab-pve01-app-crypto-portfolio-20260713T075801Z TWO_CLOUDS=OK RESTORE_CHECK=OK DATABASE_INTEGRITY=ok RETENTION=OK ROLLBACK_COUNT=3 DANGLING_IMAGES=2 REFERENCED_DANGLING_IMAGES=2 UNREFERENCED_DANGLING_IMAGES=0 PRUNED_UNREFERENCED_DANGLING_IMAGES=0 CHART_HEALTH=OK PUBLIC=OK MONITOR_TIMER=active BACKUP_TIMER=active CLOSURE_DESIRED_STATE_HEAD=2999910549eaff9b9431b9c656d318340eb95cbd DESIRED_STATE=OK PVE01_UNRELATED_FAILED_BASELINE_UNCHANGED=YES ALL_CRYPTO_PORTFOLIO_UI_TAILS_CLOSED=YES
72:1194:/var/lib/homelab-health/backup-matrix-cluster-preseal-convergence.txt:1:STATUS=OK TS=2026-07-13T17:48:14Z TYPE=backup-matrix-cluster-preseal-convergence BAD=0 MATRIX_STATUS=OK JOBS_TOTAL=35 JOBS_BAD=0 JOBS_UNMAPPED=0 JOBS_DISABLED=4 CLUSTER_PASSPORT=OK FINAL_READINESS=OK OVERALL=OK CLUSTER_ADMIN=OK PROBE_OK=4 PROBE_BAD=0 WEBPANEL=OK FAILED_UNITS=0 DESIRED_STATE_HEAD=af26ec3c4829b1330d49656205ac580ce1706263 DESIRED_STATE_DIRTY=0 MATRIX_SHA256=7613bcb25d32e61fac114da13af81e38354ec22a012a6deeaa168ee4cc454f87 MATRIX_STATE_SHA256=3791fa78cdfb56cd7027f89350dff245abcffc4f935adfc336ceeea00854ac89 CLUSTER_PASSPORT_SHA256=1b8b924ae0e5ce6872fe5671318094ce494806c2db7efe28299b5ff49adc0d0d FINAL_READINESS_SHA256=ac5159fa8c497afe021b5cb232f08cfec0e1a4d516d8aae966ba03f7034f1412 OVERALL_SHA256=6ae529bfdf5811f686d3b53978f453ce4f8757c74fc1be105de219b13a61ba46 BACKUP_RUN=NO RESTORE_RUN=NO CLOUD_WRITE_RUN=NO RETENTION_RUN=NO EVIDENCE_ROOT=/var/lib/homelab-change-sets/backup-matrix-cluster-preseal-resume/20260713T174748Z
73:1202:/var/lib/homelab-health/node-loss-final-closure.txt:1:STATUS=OK TS=2026-07-08T16:35:56Z TYPE=node-loss-final-closure BAD=0 MODE=ALL_NODE_LOSS_DRILLS_SEALED PVE03_EDGE_RESTORE=OK PVE03_EDGE_FRESHNESS=OK PVE02_FORUM_RUNTIME=OK PVE01_NEXTCLOUD_DNS=OK PVE01_CORE_MONITORING_APPLEVEL=OK FULL_VM_GAP_VM170_VM171=ACKNOWLEDGED_APP_LEVEL_ONLY TEST_VMS_CLEANED=YES DNS_PAIR_OK=YES FAILED_UNITS=pve01:0,pve02:0,pve03:0,edge:0 DIRTY_COUNT=0 REPORT=/var/lib/homelab-node-loss/final-closure-latest.md
74:1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed
75:1332:/var/lib/homelab-health/codevipe-cloud-backup-reset.txt:1:STATUS=OK TS=2026-07-08T05:25:22Z TYPE=remote-failed-unit-codevipe-reset BAD=0 WARN=0 MODE=RESET_STALE_FAILED_STATE UNIT=codevipe-cloud-backup.service REASON=last_journal_success_and_no_timer ACTION=reset_failed_only
76-1426:● homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image
77:1499:2026-07-21T11:35:23Z	BACKUP-DUAL-CLOUD-PREFLIGHT-059	c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479	1	FAIL	backup-audit
78:1500:2026-07-21T12:28:26Z	BACKUP-VM-DUMP-PIPELINE-073	cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b	1	FAIL	backup-audit
79:1501:2026-07-21T12:42:53Z	BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075	a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936	143	FAIL	backup-audit
80:1502:2026-07-21T12:56:52Z	BACKUP-DRAIN-LOCAL-ARCHIVES-079	531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32	75	FAIL	backup-remediation
81:1503:2026-07-21T13:04:48Z	BACKUP-FINAL-PROOF-082	9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601	127	FAIL	backup-audit
82:1505:2026-07-21T13:14:27Z	CLUSTER-BACKUP-POSTCLEAN-PROOF-086	601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da	127	FAIL	backup-cleanup
83:1506:2026-07-21T13:15:21Z	BACKUP-POLICY-CLOSEOUT-087	d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52	1	FAIL	backup-cleanup
84:1507:2026-07-21T13:17:20Z	BACKUP-CLEANUP-FINAL-CLOSEOUT-088	5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6	1	FAIL	backup-cleanup
85-=== ACTIONABLE_P1 ===
86-26:1:# HOMELAB ASSISTANT ERROR REGISTER
87-28:77:21. Ошибка: nested PHP php -r дал Parse error на forum-prod.
88-29:83:Команда 108 дала PHP Parse error на smtpHost и пустой proof-файл.
--
111-149:761:- The wrapper validation count is cascading missing-marker evidence and does not represent twenty-one independent faults.
112-153:774:- The reported validation count is cascading missing-marker evidence, not twenty-five independent design defects.
113:155:787:- The executed backup script does not write that health file; therefore the stale health timestamp was not evidence of backup failure.
114-171:839:- Symptom: Bash получил одиночный фрагмент и сообщил syntax error near unexpected token `(`.
115-180:865:- Symptom: error-register candidate construction stopped with RC1 before applying the candidate.
116-183:876:- Symptom: the operator shell rejected the generated one-line wrapper with syntax error near the final subshell parenthesis.
117-201:948:BACKUP-AUDIT-CONTROLPLANE-054 was successfully published at commit 2b966acfe553b4d9f8d4a62bf9533478bdd1a073, but was incorrectly reported missing because a stale directory listing was treated as authoritative.
118:203:952:BACKUP-UNSAFE-SOURCE-CAPTURE-071 failed before execution: shell syntax error near unexpected token (.
119-220:- Four Uptime Kuma monitors were still unavailable because three health endpoints returned 404 and Paperless Restore had STATUS=ERROR.
120-235:- Edge Dockge has stale inactive stack definitions after services moved to core-apps and monitoring.
121-236:- Do not delete containers. First connect remote hosts or archive stale stack definitions after classification.
122-245:- Stale edge stack definitions were not archived yet; verify Dockge UI connected state first.
--
129-459:STATUS=FAIL
130-475:JOB=filebrowser-restore-validate STATUS=STALE EVIDENCE=artifact_or_health AGE=1016424 MAX_AGE=864000
131:486:JOB=homelab-backup-sla-health STATUS=FAIL EVIDENCE=artifact_or_health AGE=144 MAX_AGE=172800
132-495:JOB=homelab-mail-cloud-restore-drill STATUS=STALE EVIDENCE=artifact_or_health AGE=988269 MAX_AGE=864000
133:508:JOB=netbird-vps-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=1747896 MAX_AGE=864000
134-510:JOB=paperless-restore-dry-run STATUS=STALE EVIDENCE=artifact_or_health AGE=1001986 MAX_AGE=864000
135:513:JOB=vm160-remote-backup STATUS=FAIL EVIDENCE=artifact_or_health AGE=26607 MAX_AGE=172800
136-676:2026-07-08T09:26:36.7403618890 390 /var/lib/homelab-health/dockge-stale-stack-classification-after-agents.txt
137-681:2026-07-08T09:26:36.7406026680 390 /var/lib/homelab-health/homepage-missing-card-proposals.txt
138-684:2026-07-08T09:26:36.7408259230 390 /var/lib/homelab-health/homepage-missing-card-proposals-final.txt
139-689:2026-07-08T09:26:36.7410552890 389 /var/lib/homelab-health/dockge-stale-archive-last.txt
--
142-1127:/var/lib/homelab-health/crypto-portfolio-remove-grt-recovery-audit.txt:1:STATUS=ERROR TS=2026-07-13T05:39:07Z TYPE=crypto-portfolio-remove-grt-recovery-audit BAD=1 COMMAND_PHYSICAL_LINES=1 ROOT_CAUSE=READINESS_ENDPOINT_MISMATCH RECOVERY_VERIFIED=NO TASK_COMPLETE=NO READY_FOR_GRT_APPLY_SCRIPT_FIX=NO AUDIT_ROOT=/var/lib/homelab-change-sets/crypto-portfolio-remove-grt-recovery-audit-v2/20260713T053907Z
143-1133:/var/lib/homelab-health/final-readiness.txt:1:STATUS=WARN TS=2026-07-21T05:31:30Z TYPE=final-readiness BAD=5 PASS=48 FAIL=5 REPORT=/var/lib/homelab-final-readiness/latest.md
144:1142:/var/lib/homelab-health/vm160-remote-backup.txt:1:STATUS=FAIL
145-1152:/var/lib/homelab-health/router-hard-power.txt:1:STATUS=ERROR TS=2026-07-16T06:20:34Z MODE=cycle REMOTE=[PRIVATE_IP] RC=74
146-1162:/var/lib/homelab-health/private-vpn-hosts.txt.bak-ok-wrapper-20260624T132705Z:16:STATUS=ERROR
147-1164:/var/lib/homelab-health/paperless-restore.txt:1:STATUS=STALE
148-1165:/var/lib/homelab-health/paperless-restore.txt:20:FRESHNESS=STALE
149:1171:/var/lib/homelab-health/backup-matrix-authoritative.txt:1:STATUS=FAIL
150-1174:/var/lib/homelab-health/ungated-health-backlog.txt:1:STATUS=OK TS=2026-07-07T07:45:01Z TYPE=ungated-health-backlog BAD=0 WARN=1 MODE=POST_PVE03_ROOT_CLEANUP_TRIAGE TOTAL=8 SUPERSEDED=4 STALE=2 POLICY_GAP_NEXT_STAGE=1 ACTIONABLE_CAPACITY=1 PVE03_ROOT_USAGE_PERCENT=31 PVE03_STAGING_USAGE_PERCENT=83 REPORT=/var/lib/homelab-ungated-health-backlog/latest.md
151-1190:/var/lib/homelab-health/mail-cloud-restore-drill.txt:1:STATUS=STALE
152-1191:/var/lib/homelab-health/mail-cloud-restore-drill.txt:20:FRESHNESS=STALE
153-1192:/var/lib/homelab-health/edge-restore-proofs-offhost.txt:1:STATUS=OK TS=20260624T190040Z TYPE=edge-restore-proofs-offhost DEST_HOST=pve02 DEST_DIR=/mnt/staging/offhost/edge-restore-proofs-from-edge ITEMS=14 OK=14 ERROR=0 BYTES=226024287 SECRET_PRINTED=NO
--
156-1207:/var/lib/homelab-health/filebrowser-restore.txt:1:STATUS=STALE
157-1208:/var/lib/homelab-health/filebrowser-restore.txt:20:FRESHNESS=STALE
158:1214:/var/lib/homelab-health/prometheus-npmplus-kuma-supersede-repair-20260709T073019Z/npmplus-kuma-config-backup.txt.bak:1:STATUS=ERROR TS=20260709T072619Z TYPE=npmplus-kuma-config-backup NOTES=edge_backup_failed
159-1494:2026-07-21T08:03:07Z	GITEA-PUSH-AUTH-001	fcfc28bcc5acb6382e895d5ed7cf51d7d5a3e2f5d8f8332770c58b42039f8a94	1	FAIL	gitea-runtime
160-1495:2026-07-21T08:13:32Z	GITEA-TOKEN-CLI-HELP-003	f1d0f967eed6490ad822944a42108c82dc4b59c9fd2557d8c64624dcb6853cd4	1	FAIL	gitea-runtime
161-1496:2026-07-21T08:14:52Z	GITEA-CLI-CAPABILITIES-004	2cd6c4d28c8f646d8063105fca80b2f80d38c6a5db3a3eb76869de326b12d2d6	1	FAIL	gitea-runtime
162-1497:2026-07-21T09:49:51Z	RUNTIME-RECEIVER-STATE-027	0831002c78c26f35826cd7199995e335ceadcd39f08b9bac23882a0fbb59c8cc	127	FAIL	gitea-runtime
163-1498:2026-07-21T10:26:18Z	RUNTIME-RECEIVER-CLEANUP-036	65fbcfbce38a55723a92a8790429d6567389321f030f02cb4bf301cbe67e2531	1	FAIL	gitea-runtime
164:1499:2026-07-21T11:35:23Z	BACKUP-DUAL-CLOUD-PREFLIGHT-059	c6f4134d15a9f641ddaf67535760d0abede973dd08222a37af05612a6f580479	1	FAIL	backup-audit
165:1500:2026-07-21T12:28:26Z	BACKUP-VM-DUMP-PIPELINE-073	cd8291d0f14a37241e6062408603455d1ebd483091bebfa46c2b7a768f21308b	1	FAIL	backup-audit
166:1501:2026-07-21T12:42:53Z	BACKUP-VM-DUMP-CHANGESET-PREFLIGHT-075	a9a8a6046441de0c40a8b13bdfe96698c53a5156b4346b7ce28dc1f2362e8936	143	FAIL	backup-audit
167:1502:2026-07-21T12:56:52Z	BACKUP-DRAIN-LOCAL-ARCHIVES-079	531e1da7ce1681eed084b22ced7da87eb1ee8995f662dc7559f4f9dde3c0ac32	75	FAIL	backup-remediation
168:1503:2026-07-21T13:04:48Z	BACKUP-FINAL-PROOF-082	9e675a3cb621d0d1fcd8918b02278ab08c1a974f544ebef9a5639bf6578ee601	127	FAIL	backup-audit
169-1504:2026-07-21T13:07:09Z	CLUSTER-CLEANUP-INVENTORY-083	09d3f14ca0205936a910dc7af7991474030273a47d76fa033ea6a3185c083922	1	FAIL	cluster-cleanup
170:1505:2026-07-21T13:14:27Z	CLUSTER-BACKUP-POSTCLEAN-PROOF-086	601958c48614dbc25cc4592b111de4dec3306e53b756b275068b81a41d42b2da	127	FAIL	backup-cleanup
171:1506:2026-07-21T13:15:21Z	BACKUP-POLICY-CLOSEOUT-087	d79c1c881ebca24db09c5b4ed4b71b09a5ad338c30789c99b0644c8935072c52	1	FAIL	backup-cleanup
172:1507:2026-07-21T13:17:20Z	BACKUP-CLEANUP-FINAL-CLOSEOUT-088	5ba2893489b6f5831fc4599d3efa874d89c3fd41da7151f9c32945d11d1a23d6	1	FAIL	backup-cleanup
173-=== ACTIONABLE_STORAGE ===
174-40:150:Факт: terminal output around storage block creation showed possible paste/truncation artifact near smartctCK=FAIL.
175-454:local             dir     active       102626232        41916788        55450180   40.84%
176-=== OPEN_ERROR_REGISTER ===
=== OPEN_ERROR_REGISTER ===
=== OPEN_ERROR_REGISTER ===
NONE
=== RECENT_FAILED_COMMANDS_NORMALIZED ===
CRITICAL_FIRST_ACTION=COMPLETE
=== CLUSTER_QUORUM ===
Cluster information
-------------------
Name:             homelab
Config Version:   3
Transport:        knet
Secure auth:      on

Quorum information
------------------
Date:             Tue Jul 21 17:39:59 2026
Quorum provider:  corosync_votequorum
Nodes:            3
Node ID:          0x00000001
Ring ID:          1.9d
Quorate:          Yes

Votequorum information
----------------------
Expected votes:   3
Highest expected: 3
Total votes:      3
Quorum:           2  
Flags:            Quorate 

Membership information
----------------------
    Nodeid      Votes Name
0x00000001          1 [PRIVATE_IP] (local)
0x00000002          1 [PRIVATE_IP]
0x00000003          1 [PRIVATE_IP]

Membership information
----------------------
    Nodeid      Votes Name
         1          1 pve01 (local)
         2          1 pve03
         3          1 pve02
=== FAILED_UNITS ===
  UNIT                                  LOAD   ACTIVE SUB    DESCRIPTION
● homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state
● homelab-cluster-admin-pull.service    loaded failed failed Pull cluster-admin observer health into pve01
● homelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03

Legend: LOAD   → Reflects whether the unit definition was properly loaded.
        ACTIVE → The high-level unit activation state, i.e. generalization of SUB.
        SUB    → The low-level unit activation state, values depend on unit type.

3 loaded units listed.
To show all installed unit files use 'systemctl list-unit-files'.
--- UNIT=codevipe-cloud-backup.service ---
Result=success
NRestarts=0
ExecMainCode=0
ExecMainStatus=0
Id=codevipe-cloud-backup.service
LoadState=not-found
ActiveState=inactive
SubState=dead
FragmentPath=
-- No entries --
--- UNIT=homelab-backup-matrix-refresh.service ---
Result=exit-code
NRestarts=0
ExecMainCode=1
ExecMainStatus=1
Id=homelab-backup-matrix-refresh.service
LoadState=loaded
ActiveState=failed
SubState=failed
FragmentPath=/etc/systemd/system/homelab-backup-matrix-refresh.service
Jul 21 05:46:46 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_STATUS=FAIL
Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_TOTAL=41
Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_OK=30
Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_BAD=6
Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_UNMAPPED=0
Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JOBS_DISABLED=5
Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
Jul 21 05:46:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 05:46:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 05:46:47 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Jul 21 06:06:46 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_STATUS=FAIL
Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_TOTAL=41
Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_OK=30
Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_BAD=6
Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_UNMAPPED=0
Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JOBS_DISABLED=5
Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
Jul 21 06:06:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 06:06:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 06:06:47 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Jul 21 06:45:16 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_STATUS=FAIL
Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_TOTAL=41
Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_OK=30
Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_BAD=6
Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_UNMAPPED=0
Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JOBS_DISABLED=5
Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
Jul 21 06:45:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 06:45:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 06:45:17 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Jul 21 07:05:43 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_STATUS=FAIL
Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_TOTAL=41
Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_OK=30
Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_BAD=6
Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_UNMAPPED=0
Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JOBS_DISABLED=5
Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
Jul 21 07:05:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 07:05:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 07:05:44 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Jul 21 07:36:59 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_STATUS=FAIL
Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_TOTAL=41
Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_OK=30
Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_BAD=6
Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_UNMAPPED=0
Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JOBS_DISABLED=5
Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
Jul 21 07:37:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 07:37:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 07:37:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Jul 21 08:07:26 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_STATUS=FAIL
Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_TOTAL=41
Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_OK=30
Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_BAD=6
Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_UNMAPPED=0
Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JOBS_DISABLED=5
Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
Jul 21 08:07:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 08:07:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 08:07:27 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Jul 21 08:30:13 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_STATUS=FAIL
Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_TOTAL=41
Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_OK=30
Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_BAD=6
Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_UNMAPPED=0
Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JOBS_DISABLED=5
Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
Jul 21 08:30:14 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 08:30:14 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 08:30:14 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Jul 21 08:41:14 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_STATUS=FAIL
Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_TOTAL=41
Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_OK=30
Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_BAD=6
Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_UNMAPPED=0
Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JOBS_DISABLED=5
Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
Jul 21 08:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 08:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 08:41:15 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Jul 21 08:50:15 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_STATUS=FAIL
Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_TOTAL=41
Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_OK=30
Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_BAD=6
Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_UNMAPPED=0
Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JOBS_DISABLED=5
Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
Jul 21 08:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 08:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 08:50:16 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
Jul 21 10:02:41 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_STATUS=FAIL
Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_TOTAL=41
Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_OK=30
Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_BAD=6
Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_UNMAPPED=0
Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JOBS_DISABLED=5
Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
Jul 21 10:02:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 10:02:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 10:02:42 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
--- UNIT=homelab-cluster-admin-incident-engine-backup.service ---
Result=success
NRestarts=0
ExecMainCode=1
ExecMainStatus=0
Id=homelab-cluster-admin-incident-engine-backup.service
LoadState=loaded
ActiveState=inactive
SubState=dead
FragmentPath=/etc/systemd/system/homelab-cluster-admin-incident-engine-backup.service
Jul 13 07:33:14 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 13 07:33:24 pve01 homelab-cluster-admin-incident-engine-backup-run[2386922]: FLAT_CHUNKED_COPY_OK SIZE=1818333 PARTS=1
Jul 13 07:33:28 pve01 homelab-cluster-admin-incident-engine-backup-run[2387061]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T043314Z
Jul 13 07:33:37 pve01 homelab-cluster-admin-incident-engine-backup-run[2387287]: FLAT_CHUNKED_COPY_OK SIZE=1818333 PARTS=1
Jul 13 07:33:38 pve01 homelab-cluster-admin-incident-engine-backup-run[2387448]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260713T043314Z
Jul 13 07:33:39 pve01 homelab-cluster-admin-incident-engine-backup-run[2386743]: [SUCCESS] short flat application uploaded and verified
Jul 13 07:33:39 pve01 homelab-cluster-admin-incident-engine-backup-run[2386712]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 13 07:33:39 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 13 07:33:39 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 13 07:33:39 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.357s CPU time, 50.3M memory peak.
Jul 14 04:46:45 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 14 04:46:58 pve01 homelab-cluster-admin-incident-engine-backup-run[2275200]: FLAT_CHUNKED_COPY_OK SIZE=2493105 PARTS=1
Jul 14 04:46:58 pve01 homelab-cluster-admin-incident-engine-backup-run[2275366]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T014645Z
Jul 14 04:47:12 pve01 homelab-cluster-admin-incident-engine-backup-run[2275633]: FLAT_CHUNKED_COPY_OK SIZE=2493105 PARTS=1
Jul 14 04:47:12 pve01 homelab-cluster-admin-incident-engine-backup-run[2275815]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T014645Z
Jul 14 04:47:18 pve01 homelab-cluster-admin-incident-engine-backup-run[2275029]: [SUCCESS] short flat application uploaded and verified
Jul 14 04:47:18 pve01 homelab-cluster-admin-incident-engine-backup-run[2274993]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 14 04:47:18 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 14 04:47:18 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 14 04:47:18 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.491s CPU time, 54.3M memory peak.
Jul 14 04:54:18 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 14 04:54:33 pve01 homelab-cluster-admin-incident-engine-backup-run[2297844]: FLAT_CHUNKED_COPY_OK SIZE=2505726 PARTS=1
Jul 14 04:54:33 pve01 homelab-cluster-admin-incident-engine-backup-run[2298021]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T015418Z
Jul 14 04:54:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2298273]: FLAT_CHUNKED_COPY_OK SIZE=2505726 PARTS=1
Jul 14 04:54:50 pve01 homelab-cluster-admin-incident-engine-backup-run[2298435]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T015418Z
Jul 14 04:54:51 pve01 homelab-cluster-admin-incident-engine-backup-run[2297589]: [SUCCESS] short flat application uploaded and verified
Jul 14 04:54:51 pve01 homelab-cluster-admin-incident-engine-backup-run[2297401]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 14 04:54:51 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 14 04:54:51 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 14 04:54:51 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.453s CPU time, 55M memory peak.
Jul 14 07:32:36 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 14 07:32:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2862512]: FLAT_CHUNKED_COPY_OK SIZE=2584458 PARTS=1
Jul 14 07:32:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2862665]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T043236Z
Jul 14 07:33:02 pve01 homelab-cluster-admin-incident-engine-backup-run[2862911]: FLAT_CHUNKED_COPY_OK SIZE=2584458 PARTS=1
Jul 14 07:33:05 pve01 homelab-cluster-admin-incident-engine-backup-run[2863090]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T043236Z
Jul 14 07:33:06 pve01 homelab-cluster-admin-incident-engine-backup-run[2862304]: [SUCCESS] short flat application uploaded and verified
Jul 14 07:33:06 pve01 homelab-cluster-admin-incident-engine-backup-run[2862165]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 14 07:33:06 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 14 07:33:06 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 14 07:33:06 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.436s CPU time, 54.9M memory peak.
Jul 14 09:43:27 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 14 09:43:39 pve01 homelab-cluster-admin-incident-engine-backup-run[3315449]: FLAT_CHUNKED_COPY_OK SIZE=2651545 PARTS=1
Jul 14 09:43:42 pve01 homelab-cluster-admin-incident-engine-backup-run[3315701]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T064327Z
Jul 14 09:43:51 pve01 homelab-cluster-admin-incident-engine-backup-run[3315929]: FLAT_CHUNKED_COPY_OK SIZE=2651545 PARTS=1
Jul 14 09:43:52 pve01 homelab-cluster-admin-incident-engine-backup-run[3316098]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260714T064327Z
Jul 14 09:43:55 pve01 homelab-cluster-admin-incident-engine-backup-run[3315291]: [SUCCESS] short flat application uploaded and verified
Jul 14 09:43:55 pve01 homelab-cluster-admin-incident-engine-backup-run[3315256]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 14 09:43:55 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 14 09:43:55 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 14 09:43:55 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.483s CPU time, 55.4M memory peak.
Jul 15 07:35:00 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 15 07:35:11 pve01 homelab-cluster-admin-incident-engine-backup-run[3512923]: FLAT_CHUNKED_COPY_OK SIZE=3303993 PARTS=1
Jul 15 07:35:11 pve01 homelab-cluster-admin-incident-engine-backup-run[3513093]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260715T043500Z
Jul 15 07:35:30 pve01 homelab-cluster-admin-incident-engine-backup-run[3513650]: FLAT_CHUNKED_COPY_OK SIZE=3303993 PARTS=1
Jul 15 07:35:31 pve01 homelab-cluster-admin-incident-engine-backup-run[3513799]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260715T043500Z
Jul 15 07:35:34 pve01 homelab-cluster-admin-incident-engine-backup-run[3512747]: [SUCCESS] short flat application uploaded and verified
Jul 15 07:35:34 pve01 homelab-cluster-admin-incident-engine-backup-run[3512710]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 15 07:35:34 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 15 07:35:34 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 15 07:35:34 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.396s CPU time, 59.3M memory peak.
Jul 16 07:32:14 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 16 07:32:23 pve01 homelab-cluster-admin-incident-engine-backup-run[95619]: FLAT_CHUNKED_COPY_OK SIZE=4089435 PARTS=1
Jul 16 07:32:24 pve01 homelab-cluster-admin-incident-engine-backup-run[95775]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260716T043214Z
Jul 16 07:32:34 pve01 homelab-cluster-admin-incident-engine-backup-run[95997]: FLAT_CHUNKED_COPY_OK SIZE=4089435 PARTS=1
Jul 16 07:32:35 pve01 homelab-cluster-admin-incident-engine-backup-run[96153]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260716T043214Z
Jul 16 07:32:36 pve01 homelab-cluster-admin-incident-engine-backup-run[95482]: [SUCCESS] short flat application uploaded and verified
Jul 16 07:32:36 pve01 homelab-cluster-admin-incident-engine-backup-run[95370]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 16 07:32:36 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 16 07:32:36 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 16 07:32:36 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.646s CPU time, 66.1M memory peak.
-- Boot 9fe27497bd1642a386cb88c33185f42f --
Jul 17 07:30:49 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 17 07:31:02 pve01 homelab-cluster-admin-incident-engine-backup-run[4051907]: FLAT_CHUNKED_COPY_OK SIZE=4782437 PARTS=1
Jul 17 07:31:03 pve01 homelab-cluster-admin-incident-engine-backup-run[4052105]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260717T043049Z
Jul 17 07:31:17 pve01 homelab-cluster-admin-incident-engine-backup-run[4052291]: FLAT_CHUNKED_COPY_OK SIZE=4782437 PARTS=1
Jul 17 07:31:20 pve01 homelab-cluster-admin-incident-engine-backup-run[4052752]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260717T043049Z
Jul 17 07:31:24 pve01 homelab-cluster-admin-incident-engine-backup-run[4051765]: [SUCCESS] short flat application uploaded and verified
Jul 17 07:31:24 pve01 homelab-cluster-admin-incident-engine-backup-run[4051685]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 17 07:31:24 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 17 07:31:24 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 17 07:31:24 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.642s CPU time, 68.6M memory peak.
Jul 18 07:31:41 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 18 07:31:54 pve01 homelab-cluster-admin-incident-engine-backup-run[685264]: FLAT_CHUNKED_COPY_OK SIZE=5630582 PARTS=1
Jul 18 07:31:55 pve01 homelab-cluster-admin-incident-engine-backup-run[685475]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260718T043141Z
Jul 18 07:32:06 pve01 homelab-cluster-admin-incident-engine-backup-run[685660]: FLAT_CHUNKED_COPY_OK SIZE=5630582 PARTS=1
Jul 18 07:32:07 pve01 homelab-cluster-admin-incident-engine-backup-run[685846]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260718T043141Z
Jul 18 07:32:08 pve01 homelab-cluster-admin-incident-engine-backup-run[685098]: [SUCCESS] short flat application uploaded and verified
Jul 18 07:32:08 pve01 homelab-cluster-admin-incident-engine-backup-run[684962]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 18 07:32:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 18 07:32:08 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 18 07:32:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.695s CPU time, 73.5M memory peak.
Jul 19 07:26:05 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 19 07:26:14 pve01 homelab-cluster-admin-incident-engine-backup-run[1415764]: FLAT_CHUNKED_COPY_OK SIZE=6386016 PARTS=1
Jul 19 07:26:14 pve01 homelab-cluster-admin-incident-engine-backup-run[1415903]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260719T042605Z
Jul 19 07:26:31 pve01 homelab-cluster-admin-incident-engine-backup-run[1416145]: FLAT_CHUNKED_COPY_OK SIZE=6386016 PARTS=1
Jul 19 07:26:31 pve01 homelab-cluster-admin-incident-engine-backup-run[1416534]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260719T042605Z
Jul 19 07:26:32 pve01 homelab-cluster-admin-incident-engine-backup-run[1415638]: [SUCCESS] short flat application uploaded and verified
Jul 19 07:26:32 pve01 homelab-cluster-admin-incident-engine-backup-run[1415567]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 19 07:26:32 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 19 07:26:32 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 19 07:26:32 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.704s CPU time, 78.3M memory peak.
Jul 20 07:37:35 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 20 07:37:49 pve01 homelab-cluster-admin-incident-engine-backup-run[2212986]: FLAT_CHUNKED_COPY_OK SIZE=7183272 PARTS=1
Jul 20 07:37:52 pve01 homelab-cluster-admin-incident-engine-backup-run[2213181]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260720T043735Z
Jul 20 07:38:04 pve01 homelab-cluster-admin-incident-engine-backup-run[2213436]: FLAT_CHUNKED_COPY_OK SIZE=7183272 PARTS=1
Jul 20 07:38:07 pve01 homelab-cluster-admin-incident-engine-backup-run[2213580]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260720T043735Z
Jul 20 07:38:08 pve01 homelab-cluster-admin-incident-engine-backup-run[2212839]: [SUCCESS] short flat application uploaded and verified
Jul 20 07:38:08 pve01 homelab-cluster-admin-incident-engine-backup-run[2212719]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 20 07:38:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 20 07:38:08 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 20 07:38:08 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.798s CPU time, 83M memory peak.
Jul 21 07:36:14 pve01 systemd[1]: Starting homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup...
Jul 21 07:36:26 pve01 homelab-cluster-admin-incident-engine-backup-run[3010900]: FLAT_CHUNKED_COPY_OK SIZE=8130454 PARTS=1
Jul 21 07:36:27 pve01 homelab-cluster-admin-incident-engine-backup-run[3011053]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260721T043614Z
Jul 21 07:36:36 pve01 homelab-cluster-admin-incident-engine-backup-run[3011255]: FLAT_CHUNKED_COPY_OK SIZE=8130454 PARTS=1
Jul 21 07:36:37 pve01 homelab-cluster-admin-incident-engine-backup-run[3011413]: FLAT_STAGE_VERIFY_OK JOB=homelab-pve01-app-cluster-admin-incident-engine-20260721T043614Z
Jul 21 07:36:40 pve01 homelab-cluster-admin-incident-engine-backup-run[3010739]: [SUCCESS] short flat application uploaded and verified
Jul 21 07:36:40 pve01 homelab-cluster-admin-incident-engine-backup-run[3010566]: CLUSTER_ADMIN_INCIDENT_ENGINE_BACKUP_RUN=OK
Jul 21 07:36:40 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Deactivated successfully.
Jul 21 07:36:40 pve01 systemd[1]: Finished homelab-cluster-admin-incident-engine-backup.service - Cluster Admin Incident Engine encrypted two-cloud backup.
Jul 21 07:36:40 pve01 systemd[1]: homelab-cluster-admin-incident-engine-backup.service: Consumed 3.760s CPU time, 88.3M memory peak.
--- UNIT=homelab-cluster-admin-pull.service ---
Result=exit-code
NRestarts=0
ExecMainCode=1
ExecMainStatus=1
Id=homelab-cluster-admin-pull.service
LoadState=loaded
ActiveState=failed
SubState=failed
FragmentPath=/etc/systemd/system/homelab-cluster-admin-pull.service
Jul 21 01:06:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 01:11:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 01:11:36 pve01 homelab-cluster-admin-pull[1709417]: STATUS=WARN TS=2026-07-20T22:11:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 01:11:36 pve01 homelab-cluster-admin-pull[1709418]: STATUS=WARN TS=2026-07-20T22:11:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 01:11:36 pve01 homelab-cluster-admin-pull[1709419]: STATUS=WARN TS=2026-07-20T22:11:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 01:11:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:11:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 01:11:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 01:16:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 01:16:36 pve01 homelab-cluster-admin-pull[1733408]: STATUS=WARN TS=2026-07-20T22:16:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 01:16:36 pve01 homelab-cluster-admin-pull[1733409]: STATUS=WARN TS=2026-07-20T22:16:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 01:16:36 pve01 homelab-cluster-admin-pull[1733410]: STATUS=WARN TS=2026-07-20T22:16:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 01:16:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:16:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 01:16:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 01:21:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 01:21:36 pve01 homelab-cluster-admin-pull[1749007]: STATUS=WARN TS=2026-07-20T22:21:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 01:21:36 pve01 homelab-cluster-admin-pull[1749008]: STATUS=WARN TS=2026-07-20T22:21:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 01:21:36 pve01 homelab-cluster-admin-pull[1749009]: STATUS=WARN TS=2026-07-20T22:21:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 01:21:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:21:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 01:21:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 01:26:30 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 01:26:36 pve01 homelab-cluster-admin-pull[1764598]: STATUS=WARN TS=2026-07-20T22:26:30Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 01:26:36 pve01 homelab-cluster-admin-pull[1764599]: STATUS=WARN TS=2026-07-20T22:26:30Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 01:26:36 pve01 homelab-cluster-admin-pull[1764600]: STATUS=WARN TS=2026-07-20T22:26:30Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 01:26:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:26:36 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 01:26:36 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 01:31:45 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 01:31:52 pve01 homelab-cluster-admin-pull[1788839]: STATUS=WARN TS=2026-07-20T22:31:45Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 01:31:52 pve01 homelab-cluster-admin-pull[1788840]: STATUS=WARN TS=2026-07-20T22:31:45Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 01:31:52 pve01 homelab-cluster-admin-pull[1788841]: STATUS=WARN TS=2026-07-20T22:31:45Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 01:31:52 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:31:52 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 01:31:52 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 01:37:07 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 01:37:07 pve01 homelab-cluster-admin-pull[1804555]: STATUS=WARN TS=2026-07-20T22:37:07Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 01:37:07 pve01 homelab-cluster-admin-pull[1804556]: STATUS=WARN TS=2026-07-20T22:37:07Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 01:37:07 pve01 homelab-cluster-admin-pull[1804557]: STATUS=WARN TS=2026-07-20T22:37:07Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 01:37:07 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:37:07 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 01:37:07 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 01:42:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 01:42:15 pve01 homelab-cluster-admin-pull[1820229]: STATUS=WARN TS=2026-07-20T22:42:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 01:42:15 pve01 homelab-cluster-admin-pull[1820230]: STATUS=WARN TS=2026-07-20T22:42:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 01:42:15 pve01 homelab-cluster-admin-pull[1820231]: STATUS=WARN TS=2026-07-20T22:42:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 01:42:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:42:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 01:42:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 01:47:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 01:47:15 pve01 homelab-cluster-admin-pull[1844123]: STATUS=WARN TS=2026-07-20T22:47:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 01:47:15 pve01 homelab-cluster-admin-pull[1844124]: STATUS=WARN TS=2026-07-20T22:47:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 01:47:15 pve01 homelab-cluster-admin-pull[1844125]: STATUS=WARN TS=2026-07-20T22:47:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 01:47:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:47:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 01:47:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 01:52:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 01:52:15 pve01 homelab-cluster-admin-pull[1859657]: STATUS=WARN TS=2026-07-20T22:52:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 01:52:15 pve01 homelab-cluster-admin-pull[1859658]: STATUS=WARN TS=2026-07-20T22:52:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 01:52:15 pve01 homelab-cluster-admin-pull[1859659]: STATUS=WARN TS=2026-07-20T22:52:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 01:52:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:52:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 01:52:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 01:57:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 01:57:15 pve01 homelab-cluster-admin-pull[1875102]: STATUS=WARN TS=2026-07-20T22:57:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 01:57:15 pve01 homelab-cluster-admin-pull[1875103]: STATUS=WARN TS=2026-07-20T22:57:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 01:57:15 pve01 homelab-cluster-admin-pull[1875104]: STATUS=WARN TS=2026-07-20T22:57:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 01:57:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:57:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 01:57:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 02:02:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 02:02:15 pve01 homelab-cluster-admin-pull[1899107]: STATUS=WARN TS=2026-07-20T23:02:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 02:02:15 pve01 homelab-cluster-admin-pull[1899108]: STATUS=WARN TS=2026-07-20T23:02:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 02:02:15 pve01 homelab-cluster-admin-pull[1899109]: STATUS=WARN TS=2026-07-20T23:02:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 02:02:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:02:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 02:02:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 02:07:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 02:07:15 pve01 homelab-cluster-admin-pull[1914657]: STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 02:07:15 pve01 homelab-cluster-admin-pull[1914658]: STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 02:07:15 pve01 homelab-cluster-admin-pull[1914659]: STATUS=WARN TS=2026-07-20T23:07:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 02:07:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:07:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 02:07:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 02:12:14 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 02:12:15 pve01 homelab-cluster-admin-pull[1930715]: STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 02:12:15 pve01 homelab-cluster-admin-pull[1930716]: STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 02:12:15 pve01 homelab-cluster-admin-pull[1930717]: STATUS=WARN TS=2026-07-20T23:12:14Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 02:12:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:12:15 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 02:12:15 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 02:14:04 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 02:14:11 pve01 homelab-cluster-admin-pull[1932800]: STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 02:14:11 pve01 homelab-cluster-admin-pull[1932801]: STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 02:14:11 pve01 homelab-cluster-admin-pull[1932802]: STATUS=WARN TS=2026-07-20T23:14:04Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 02:14:11 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:14:11 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 02:14:11 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 02:20:13 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 02:20:20 pve01 homelab-cluster-admin-pull[1958372]: STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 02:20:20 pve01 homelab-cluster-admin-pull[1958373]: STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=3 PROBE_BAD=1 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 02:20:20 pve01 homelab-cluster-admin-pull[1958374]: STATUS=WARN TS=2026-07-20T23:20:13Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=3 PROBE_BAD=1 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 02:20:20 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:20:20 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 02:20:20 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 02:35:05 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 02:35:12 pve01 homelab-cluster-admin-pull[2013630]: STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 02:35:12 pve01 homelab-cluster-admin-pull[2013631]: STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 02:35:12 pve01 homelab-cluster-admin-pull[2013632]: STATUS=WARN TS=2026-07-20T23:35:05Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 02:35:12 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:35:12 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 02:35:12 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
Jul 21 02:36:57 pve01 systemd[1]: Starting homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01...
Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015088]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-central-pull BAD=1 VMID=180 IP=[PRIVATE_IP] SSH_RC=0 OBSERVER_STATUS=WARN REPORT=/var/lib/homelab-cluster-admin/observer-remote-latest.txt
Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015089]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-full-observer-central BAD=1 WARN=0 VMID=180 PULL_RC=0 FULL_STATUS=WARN PROBE_OK=2 PROBE_BAD=2 REPORT=/var/lib/homelab-cluster-admin/full-observer-central-latest.md
Jul 21 02:37:04 pve01 homelab-cluster-admin-pull[2015090]: STATUS=WARN TS=2026-07-20T23:36:57Z TYPE=cluster-admin-restricted-probes BAD=1 VMID=180 MODE=FORCED_COMMAND_READONLY_CURRENT PROBE_OK=2 PROBE_BAD=2 TARGETS=pve01,pve02,pve03,edge SOURCE=full-observer
Jul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:37:04 pve01 systemd[1]: homelab-cluster-admin-pull.service: Failed with result 'exit-code'.
Jul 21 02:37:04 pve01 systemd[1]: Failed to start homelab-cluster-admin-pull.service - Pull cluster-admin observer health into pve01.
--- UNIT=homelab-edge-vm-offhost-freshness.timer ---
Result=success
Id=homelab-edge-vm-offhost-freshness.timer
LoadState=loaded
ActiveState=active
SubState=waiting
FragmentPath=/etc/systemd/system/homelab-edge-vm-offhost-freshness.timer
Jul 16 09:20:43 pve01 systemd[1]: homelab-edge-vm-offhost-freshness.timer: Deactivated successfully.
Jul 16 09:20:43 pve01 systemd[1]: Stopped homelab-edge-vm-offhost-freshness.timer - Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump.
-- Boot 9fe27497bd1642a386cb88c33185f42f --
Jul 16 11:38:48 pve01 systemd[1]: Started homelab-edge-vm-offhost-freshness.timer - Run Homelab edge-vm VM130 offhost freshness sync daily after nightly vzdump.
--- UNIT=homelab-immutable-offline-verify.service ---
Result=success
NRestarts=0
ExecMainCode=0
ExecMainStatus=0
Id=homelab-immutable-offline-verify.service
LoadState=not-found
ActiveState=inactive
SubState=dead
FragmentPath=
-- No entries --
--- UNIT=homelab-vm160-remote-backup.service ---
Result=exit-code
NRestarts=0
ExecMainCode=1
ExecMainStatus=1
Id=homelab-vm160-remote-backup.service
LoadState=loaded
ActiveState=failed
SubState=failed
FragmentPath=/etc/systemd/system/homelab-vm160-remote-backup.service
Jul 18 01:02:41 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
Jul 18 01:15:43 pve01 homelab-vm160-remote-backup[3491061]: VM160_TARGET_FINALIZE_STATUS=OK
Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3530071]: VM160_TARGET_PROMOTE_STATUS=OK
Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: VM160_REMOTE_BACKUP_STATUS=OK
Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z
Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T220242Z/vzdump-qemu-160-20260717T220242Z.vma.zst
Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: SIZE_BYTES=84993482488
Jul 18 01:19:10 pve01 homelab-vm160-remote-backup[3473847]: SHA256=a9562d8b60d7fd5e7ec73bc43d252cfea80ed12c05b32cca0a73d7bb88532411
Jul 18 01:19:10 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully.
Jul 18 01:19:10 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Jul 18 01:19:10 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.280s CPU time, 15.8M memory peak.
Jul 18 01:38:18 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
Jul 18 01:51:18 pve01 homelab-vm160-remote-backup[3609200]: VM160_TARGET_FINALIZE_STATUS=OK
Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3648126]: VM160_TARGET_PROMOTE_STATUS=OK
Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: VM160_REMOTE_BACKUP_STATUS=OK
Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: TARGET_DIR=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z
Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: ARCHIVE=/mnt/staging/vzdump/vm160-pve02-20260717T223819Z/vzdump-qemu-160-20260717T223819Z.vma.zst
Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: SIZE_BYTES=84995216465
Jul 18 01:54:48 pve01 homelab-vm160-remote-backup[3592132]: SHA256=257e10821e62e3e2f9318b238a5302a6db601dd597bfa3e0d77aba07f3b85e72
Jul 18 01:54:48 pve01 systemd[1]: homelab-vm160-remote-backup.service: Deactivated successfully.
Jul 18 01:54:48 pve01 systemd[1]: Finished homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Jul 18 01:54:48 pve01 systemd[1]: homelab-vm160-remote-backup.service: Consumed 2min 44.842s CPU time, 16.1M memory peak.
Jul 18 04:16:25 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
Jul 18 04:16:27 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 18 04:16:27 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 18 04:16:27 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Jul 19 04:19:14 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
Jul 19 04:19:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 04:19:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 19 04:19:16 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Jul 20 04:14:14 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
Jul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 20 04:14:16 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Jul 20 19:28:57 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
Jul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 20 19:28:59 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Jul 21 02:14:11 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
Jul 21 02:14:13 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:14:13 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 21 02:14:13 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Jul 21 02:20:20 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
Jul 21 02:20:21 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:20:21 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 21 02:20:21 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
Jul 21 02:39:12 pve01 systemd[1]: Starting homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03...
Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 21 02:39:14 pve01 systemd[1]: Failed to start homelab-vm160-remote-backup.service - VM160 full-image backup streamed from pve02 to pve03.
=== PROXMOX_STORAGE ===
Name             Type     Status     Total (KiB)      Used (KiB) Available (KiB)        %
local             dir     active       102626232        41936392        55430576   40.86%
local-lvm     lvmthin     active       848797696       280188119       568609576   33.01%
Filesystem                       Type      Size  Used Avail Use% Mounted on
udev                             devtmpfs   15G     0   15G   0% /dev
tmpfs                            tmpfs     3.2G  3.8M  3.2G   1% /run
/dev/mapper/pve-root             ext4       98G   40G   53G  44% /
tmpfs                            tmpfs      16G   66M   16G   1% /dev/shm
efivarfs                         efivarfs  192K  135K   53K  72% /sys/firmware/efi/efivars
tmpfs                            tmpfs     5.0M     0  5.0M   0% /run/lock
tmpfs                            tmpfs     1.0M     0  1.0M   0% /run/credentials/systemd-journald.service
tmpfs                            tmpfs      16G   48M   16G   1% /tmp
/dev/nvme0n1p2                   vfat     1022M  9.1M 1013M   1% /boot/efi
/dev/mapper/pve-homelab--staging ext4      295G  148G  133G  53% /mnt/staging
/dev/fuse                        fuse      128M  1.4M  127M   2% /etc/pve
shm                              tmpfs      63M     0   63M   0% /var/lib/containers/storage/overlay-containers/fbb240ea5b547fa140d8edbc3b16daaf4b28a21c7ee424abedab068865a95d5b/userdata/shm
overlay                          overlay    98G   40G   53G  44% /var/lib/containers/storage/overlay/4f68ccfa2802b35db2a4c6b396e477f756cb85aa42fa818dd111745ec503616b/merged
tmpfs                            tmpfs     1.0M     0  1.0M   0% /run/credentials/getty@tty1.service
tmpfs                            tmpfs     3.2G  4.0K  3.2G   1% /run/user/0
Filesystem                       Inodes IUsed IFree IUse% Mounted on
udev                               3.6M   733  3.6M    1% /dev
tmpfs                              3.9M  1.4K  3.9M    1% /run
/dev/mapper/pve-root               6.3M  128K  6.2M    2% /
tmpfs                              3.9M   137  3.9M    1% /dev/shm
efivarfs                              0     0     0     - /sys/firmware/efi/efivars
tmpfs                              3.9M    32  3.9M    1% /run/lock
tmpfs                              1.0K     1  1023    1% /run/credentials/systemd-journald.service
tmpfs                              1.0M  3.5K 1021K    1% /tmp
/dev/nvme0n1p2                        0     0     0     - /boot/efi
/dev/mapper/pve-homelab--staging    19M  4.3K   19M    1% /mnt/staging
/dev/fuse                          256K    89  256K    1% /etc/pve
shm                                3.9M     1  3.9M    1% /var/lib/containers/storage/overlay-containers/fbb240ea5b547fa140d8edbc3b16daaf4b28a21c7ee424abedab068865a95d5b/userdata/shm
overlay                            6.3M  128K  6.2M    2% /var/lib/containers/storage/overlay/4f68ccfa2802b35db2a4c6b396e477f756cb85aa42fa818dd111745ec503616b/merged
tmpfs                              1.0K     1  1023    1% /run/credentials/getty@tty1.service
tmpfs                              796K    28  796K    1% /run/user/0
=== STORAGE_BACKENDS ===
no pools available
Personalities : 
unused devices: <none>
File descriptor 9 (/var/lib/homelab-chat-run/state/runner.lock) leaked on pvs invocation. Parent PID 918816: /usr/bin/bash
  /dev/nvme0n1p3 <952.00g 16.00g a-- 
File descriptor 9 (/var/lib/homelab-chat-run/state/runner.lock) leaked on vgs invocation. Parent PID 918816: /usr/bin/bash
  pve <952.00g 16.00g wz--n-
File descriptor 9 (/var/lib/homelab-chat-run/state/runner.lock) leaked on lvs invocation. Parent PID 918816: /usr/bin/bash
  data                                                    pve <809.48g twi-aotz-- 33.01  1.42  
  [data_tdata]                                            pve <809.48g Twi-ao----              
  [data_tmeta]                                            pve   <8.26g ewi-ao----              
  homelab-staging                                         pve  300.00g Vwi-aotz-- 66.86        
  [lvol0_pmspare]                                         pve   <8.26g ewi-------              
  root                                                    pve  100.00g -wi-ao----              
  snap_vm-150-disk-0_before-nc-freshcheck-20260614-233339 pve   64.00g Vri---tz-k              
  snap_vm-150-disk-0_nc-posttune-20260614-2343            pve   64.00g Vri---tz-k              
  swap                                                    pve   10.00g -wi-ao----              
  vm-110-disk-0                                           pve    8.00g Vwi-aotz-- 26.63        
  vm-112-disk-0                                           pve    8.00g Vwi-aotz-- 13.54        
  vm-150-cloudinit                                        pve    4.00m Vwi-aotz-- 9.38         
  vm-150-disk-0                                           pve   64.00g Vwi-aotz-- 22.72        
  vm-170-cloudinit                                        pve    4.00m Vwi-aotz-- 9.38         
  vm-170-disk-0                                           pve  120.00g Vwi-aotz-- 32.00        
  vm-171-cloudinit                                        pve    4.00m Vwi-aotz-- 9.38         
  vm-171-disk-0                                           pve  120.00g Vwi-aotz-- 7.05         
=== KERNEL_CRITICAL_LAST_24H ===
Jul 20 19:31:56 pve01 systemd-sysv-generator[535545]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 20 19:31:56 pve01 systemd-sysv-generator[535545]: Please update package to include a native systemd unit file.
Jul 20 19:31:56 pve01 systemd-sysv-generator[535545]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 20 19:31:56 pve01 systemd-sysv-generator[535601]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 20 19:31:56 pve01 systemd-sysv-generator[535601]: Please update package to include a native systemd unit file.
Jul 20 19:31:56 pve01 systemd-sysv-generator[535601]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 20 19:31:57 pve01 systemd-sysv-generator[535675]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 20 19:31:57 pve01 systemd-sysv-generator[535675]: Please update package to include a native systemd unit file.
Jul 20 19:31:57 pve01 systemd-sysv-generator[535675]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 20 19:31:57 pve01 systemd-sysv-generator[535733]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 20 19:31:57 pve01 systemd-sysv-generator[535733]: Please update package to include a native systemd unit file.
Jul 20 19:31:57 pve01 systemd-sysv-generator[535733]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 20 22:15:38 pve01 systemd-sysv-generator[1097452]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 20 22:15:38 pve01 systemd-sysv-generator[1097452]: Please update package to include a native systemd unit file.
Jul 20 22:15:38 pve01 systemd-sysv-generator[1097452]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 20 22:15:39 pve01 systemd-sysv-generator[1097526]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 20 22:15:39 pve01 systemd-sysv-generator[1097526]: Please update package to include a native systemd unit file.
Jul 20 22:15:39 pve01 systemd-sysv-generator[1097526]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 20 23:59:14 pve01 systemd-sysv-generator[1457483]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 20 23:59:14 pve01 systemd-sysv-generator[1457483]: Please update package to include a native systemd unit file.
Jul 20 23:59:14 pve01 systemd-sysv-generator[1457483]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 20 23:59:14 pve01 systemd-sysv-generator[1457536]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 20 23:59:14 pve01 systemd-sysv-generator[1457536]: Please update package to include a native systemd unit file.
Jul 20 23:59:14 pve01 systemd-sysv-generator[1457536]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 00:09:32 pve01 systemd-sysv-generator[1497895]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 00:09:32 pve01 systemd-sysv-generator[1497895]: Please update package to include a native systemd unit file.
Jul 21 00:09:32 pve01 systemd-sysv-generator[1497895]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 00:09:32 pve01 systemd-sysv-generator[1497977]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 00:09:32 pve01 systemd-sysv-generator[1497977]: Please update package to include a native systemd unit file.
Jul 21 00:09:32 pve01 systemd-sysv-generator[1497977]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 00:33:12 pve01 systemd-sysv-generator[1580930]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 00:33:12 pve01 systemd-sysv-generator[1580930]: Please update package to include a native systemd unit file.
Jul 21 00:33:12 pve01 systemd-sysv-generator[1580930]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 00:33:12 pve01 systemd-sysv-generator[1580983]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 00:33:12 pve01 systemd-sysv-generator[1580983]: Please update package to include a native systemd unit file.
Jul 21 00:33:12 pve01 systemd-sysv-generator[1580983]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917427]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917427]: Please update package to include a native systemd unit file.
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917427]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917497]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917497]: Please update package to include a native systemd unit file.
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917497]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917569]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917569]: Please update package to include a native systemd unit file.
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917569]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917640]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917640]: Please update package to include a native systemd unit file.
Jul 21 02:10:30 pve01 systemd-sysv-generator[1917640]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:11:53 pve01 systemd-sysv-generator[1930242]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:11:53 pve01 systemd-sysv-generator[1930242]: Please update package to include a native systemd unit file.
Jul 21 02:11:53 pve01 systemd-sysv-generator[1930242]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:11:54 pve01 systemd-sysv-generator[1930305]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:11:54 pve01 systemd-sysv-generator[1930305]: Please update package to include a native systemd unit file.
Jul 21 02:11:54 pve01 systemd-sysv-generator[1930305]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:11:54 pve01 systemd-sysv-generator[1930369]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:11:54 pve01 systemd-sysv-generator[1930369]: Please update package to include a native systemd unit file.
Jul 21 02:11:54 pve01 systemd-sysv-generator[1930369]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:14:02 pve01 systemd-sysv-generator[1932245]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:14:02 pve01 systemd-sysv-generator[1932245]: Please update package to include a native systemd unit file.
Jul 21 02:14:02 pve01 systemd-sysv-generator[1932245]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:14:03 pve01 systemd-sysv-generator[1932308]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:14:03 pve01 systemd-sysv-generator[1932308]: Please update package to include a native systemd unit file.
Jul 21 02:14:03 pve01 systemd-sysv-generator[1932308]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:14:03 pve01 systemd-sysv-generator[1932371]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:14:03 pve01 systemd-sysv-generator[1932371]: Please update package to include a native systemd unit file.
Jul 21 02:14:03 pve01 systemd-sysv-generator[1932371]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:14:04 pve01 systemd-sysv-generator[1932629]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:14:04 pve01 systemd-sysv-generator[1932629]: Please update package to include a native systemd unit file.
Jul 21 02:14:04 pve01 systemd-sysv-generator[1932629]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:14:11 pve01 systemd-sysv-generator[1932867]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:14:11 pve01 systemd-sysv-generator[1932867]: Please update package to include a native systemd unit file.
Jul 21 02:14:11 pve01 systemd-sysv-generator[1932867]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:14:13 pve01 systemd-sysv-generator[1932964]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:14:13 pve01 systemd-sysv-generator[1932964]: Please update package to include a native systemd unit file.
Jul 21 02:14:13 pve01 systemd-sysv-generator[1932964]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:16:57 pve01 systemd-sysv-generator[1955003]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:16:57 pve01 systemd-sysv-generator[1955003]: Please update package to include a native systemd unit file.
Jul 21 02:16:57 pve01 systemd-sysv-generator[1955003]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:16:57 pve01 systemd-sysv-generator[1955071]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:16:57 pve01 systemd-sysv-generator[1955071]: Please update package to include a native systemd unit file.
Jul 21 02:16:57 pve01 systemd-sysv-generator[1955071]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:16:57 pve01 systemd-sysv-generator[1955141]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:16:57 pve01 systemd-sysv-generator[1955141]: Please update package to include a native systemd unit file.
Jul 21 02:16:57 pve01 systemd-sysv-generator[1955141]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:20:11 pve01 systemd-sysv-generator[1957829]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:20:11 pve01 systemd-sysv-generator[1957829]: Please update package to include a native systemd unit file.
Jul 21 02:20:11 pve01 systemd-sysv-generator[1957829]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:20:11 pve01 systemd-sysv-generator[1957891]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:20:11 pve01 systemd-sysv-generator[1957891]: Please update package to include a native systemd unit file.
Jul 21 02:20:11 pve01 systemd-sysv-generator[1957891]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:20:12 pve01 systemd-sysv-generator[1957959]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:20:12 pve01 systemd-sysv-generator[1957959]: Please update package to include a native systemd unit file.
Jul 21 02:20:12 pve01 systemd-sysv-generator[1957959]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:20:13 pve01 systemd-sysv-generator[1958205]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:20:13 pve01 systemd-sysv-generator[1958205]: Please update package to include a native systemd unit file.
Jul 21 02:20:13 pve01 systemd-sysv-generator[1958205]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:20:20 pve01 systemd-sysv-generator[1958431]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:20:20 pve01 systemd-sysv-generator[1958431]: Please update package to include a native systemd unit file.
Jul 21 02:20:20 pve01 systemd-sysv-generator[1958431]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:20:21 pve01 systemd-sysv-generator[1958526]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:20:21 pve01 systemd-sysv-generator[1958526]: Please update package to include a native systemd unit file.
Jul 21 02:20:21 pve01 systemd-sysv-generator[1958526]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:22:02 pve01 systemd-sysv-generator[1971264]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:22:02 pve01 systemd-sysv-generator[1971264]: Please update package to include a native systemd unit file.
Jul 21 02:22:02 pve01 systemd-sysv-generator[1971264]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:22:02 pve01 systemd-sysv-generator[1971316]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:22:02 pve01 systemd-sysv-generator[1971316]: Please update package to include a native systemd unit file.
Jul 21 02:22:02 pve01 systemd-sysv-generator[1971316]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:22:02 pve01 systemd-sysv-generator[1971377]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:22:02 pve01 systemd-sysv-generator[1971377]: Please update package to include a native systemd unit file.
Jul 21 02:22:02 pve01 systemd-sysv-generator[1971377]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:23:50 pve01 systemd-sysv-generator[1973034]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:23:50 pve01 systemd-sysv-generator[1973034]: Please update package to include a native systemd unit file.
Jul 21 02:23:50 pve01 systemd-sysv-generator[1973034]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:23:50 pve01 systemd-sysv-generator[1973101]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:23:50 pve01 systemd-sysv-generator[1973101]: Please update package to include a native systemd unit file.
Jul 21 02:23:50 pve01 systemd-sysv-generator[1973101]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 02:23:50 pve01 systemd-sysv-generator[1973172]: SysV service '/etc/init.d/openipmi' lacks a native systemd unit file, automatically generating a unit file for compatibility.
Jul 21 02:23:50 pve01 systemd-sysv-generator[1973172]: Please update package to include a native systemd unit file.
Jul 21 02:23:50 pve01 systemd-sysv-generator[1973172]: ! This compatibility logic is deprecated, expect removal soon. !
Jul 21 03:30:02 pve01 kernel: EXT4-fs (dm-15): write access unavailable, skipping orphan cleanup
Jul 21 03:30:10 pve01 kernel: EXT4-fs (dm-15): write access unavailable, skipping orphan cleanup
=== BACKUP_HEALTH ===
  dev-disk-by\x2did-dm\x2dname\x2dpve\x2dvm\x2d\x2d150\x2d\x2dcloudinit.device                                                         loaded    active     plugged   /dev/disk/by-id/dm-name-pve-vm--150--cloudinit
  dev-disk-by\x2did-dm\x2dname\x2dpve\x2dvm\x2d\x2d170\x2d\x2dcloudinit.device                                                         loaded    active     plugged   /dev/disk/by-id/dm-name-pve-vm--170--cloudinit
  dev-disk-by\x2did-dm\x2dname\x2dpve\x2dvm\x2d\x2d171\x2d\x2dcloudinit.device                                                         loaded    active     plugged   /dev/disk/by-id/dm-name-pve-vm--171--cloudinit
  dev-mapper-pve\x2dvm\x2d\x2d150\x2d\x2dcloudinit.device                                                                              loaded    active     plugged   /dev/mapper/pve-vm--150--cloudinit
  dev-mapper-pve\x2dvm\x2d\x2d170\x2d\x2dcloudinit.device                                                                              loaded    active     plugged   /dev/mapper/pve-vm--170--cloudinit
  dev-mapper-pve\x2dvm\x2d\x2d171\x2d\x2dcloudinit.device                                                                              loaded    active     plugged   /dev/mapper/pve-vm--171--cloudinit
  dev-pve-vm\x2d150\x2dcloudinit.device                                                                                                loaded    active     plugged   /dev/pve/vm-150-cloudinit
  dev-pve-vm\x2d170\x2dcloudinit.device                                                                                                loaded    active     plugged   /dev/pve/vm-170-cloudinit
  dev-pve-vm\x2d171\x2dcloudinit.device                                                                                                loaded    active     plugged   /dev/pve/vm-171-cloudinit
  dpkg-db-backup.service                                                                                                               loaded    inactive   dead      Daily dpkg database backup service
  filebrowser-backup.service                                                                                                           loaded    inactive   dead      File Browser backup
  homelab-alertmanager-backup.service                                                                                                  loaded    inactive   dead      Alertmanager encrypted two-cloud application backup
  homelab-appbackup-dockge-stacks.service                                                                                              loaded    inactive   dead      Homelab app-level backup for dockge-stacks
  homelab-appbackup-mariadb-batch.service                                                                                              loaded    inactive   dead      Homelab MariaDB app batch backup
  homelab-appbackup-npmplus-edge.service                                                                                               loaded    inactive   dead      Homelab app-level backup for npmplus-edge
  homelab-appbackup-postgres-batch.service                                                                                             loaded    inactive   dead      Homelab Postgres app batch backup
  homelab-appbackup-sqlite-batch.service                                                                                               loaded    inactive   dead      Homelab SQLite app batch backup
  homelab-auto-backup.service                                                                                                          loaded    inactive   dead      Homelab automatic full backup with retention
  homelab-backup-coverage-matrix.service                                                                                               loaded    inactive   dead      Homelab backup coverage matrix
  homelab-backup-freshness.service                                                                                                     loaded    inactive   dead      Homelab backup freshness collector
  homelab-backup-healthcheck.service                                                                                                   loaded    inactive   dead      Homelab backup healthcheck
● homelab-backup-matrix-refresh.service                                                                                                loaded    failed     failed    Generate unified homelab backup matrix state
  homelab-backup-sla-health.service                                                                                                    loaded    inactive   dead      Homelab backup freshness SLA health
  homelab-beszel-backup.service                                                                                                        loaded    inactive   dead      Beszel encrypted two-cloud application backup
  homelab-blackbox-exporter-backup.service                                                                                             loaded    inactive   dead      Blackbox Exporter encrypted two-cloud application backup
  homelab-cluster-admin-incident-engine-backup.service                                                                                 loaded    inactive   dead      Cluster Admin Incident Engine encrypted two-cloud backup
  homelab-crypto-portfolio-backup.service                                                                                              loaded    inactive   dead      Gram Crypto portfolio two-cloud backup
  homelab-emergency-backup.service                                                                                                     loaded    inactive   dead      Homelab emergency/config backup to two encrypted clouds
  homelab-extended-appbackup.service                                                                                                   loaded    inactive   dead      Homelab extended P1/P2 appbackup run
  homelab-gitea-secondary-backup.service                                                                                               loaded    inactive   dead      Create and copy a verified Gitea backup to pve03
  homelab-legacy-monitoring-backups.service                                                                                            loaded    inactive   dead      Legacy monitoring applications encrypted two-cloud backup batch
  homelab-mail-cloud-critical-upload.service                                                                                           loaded    inactive   dead      Homelab Mail Cloud critical/small baseline upload
  homelab-mail-cloud-edge-vm-upload.service                                                                                            loaded    inactive   dead      Homelab Mail Cloud edge-vm large chunked upload
  homelab-mail-cloud-nextcloud-vm-upload.service                                                                                       loaded    inactive   dead      Homelab Mail Cloud Nextcloud VM vzdump upload
  homelab-mail-cloud-restore-drill.service                                                                                             loaded    inactive   dead      Homelab Mail Cloud restore drill
  homelab-router-running-config-mail-cloud.service                                                                                     loaded    inactive   dead      Homelab router running-config Mail-cloud backup
  homelab-verified-backup-health.service                                                                                               loaded    inactive   dead      Homelab verified backup health evidence
  homelab-vm-backup-health.service                                                                                                     loaded    inactive   dead      Homelab VM backup health
  homelab-vm-backup-policy.service                                                                                                     loaded    inactive   dead      Homelab VM backup policy and feasibility health
  homelab-vm-full-backup.service                                                                                                       loaded    inactive   dead      Full vzdump backup of VM170 and VM171 to offhost and two clouds
  homelab-vm-local-dumps-2cloud.service                                                                                                loaded    inactive   dead      Homelab VM/CT existing local dumps two-cloud verifier
  homelab-vm-local-dumps-retention.service                                                                                             loaded    inactive   dead      Homelab VM local dumps cloud retention
● homelab-vm160-remote-backup.service                                                                                                  loaded    failed     failed    VM160 full-image backup streamed from pve02 to pve03
  immich-media-backup.service                                                                                                          loaded    inactive   dead      Immich media backup to staging
  immich-media-offhost-sync.service                                                                                                    loaded    inactive   dead      Sync Immich media backups from pve01 to pve02
  memos-backup.service                                                                                                                 loaded    inactive   dead      Memos data backup to pve01 staging
  memos-offhost-sync.service                                                                                                           loaded    inactive   dead      Memos off-host backup sync to pve02
  postfix.service                                                                                                                      loaded    active     running   Postfix Mail Transport Agent (main/default instance)
  postfix@-.service                                                                                                                    loaded    inactive   dead      Postfix Mail Transport Agent (instance -, /etc/postfix-/)
  skladchik-reports-monitor-backup-restore-check.service                                                                               loaded    inactive   dead      Restore-check Skladchik reports monitor cloud backups
  skladchik-reports-monitor-backup.service                                                                                             loaded    inactive   dead      Backup Skladchik reports monitor from edge-vm to encrypted clouds
  dpkg-db-backup.timer                                                                                                                 loaded    active     waiting   Daily dpkg database backup timer
  filebrowser-backup.timer                                                                                                             loaded    active     waiting   Run File Browser backup
  homelab-alertmanager-backup.timer                                                                                                    loaded    active     waiting   Daily Alertmanager two-cloud backup
  homelab-appbackup-dockge-stacks.timer                                                                                                loaded    active     waiting   Daily homelab app-level backup timer for dockge-stacks
  homelab-appbackup-mariadb-batch.timer                                                                                                loaded    active     waiting   Daily Homelab MariaDB app batch backup timer
  homelab-appbackup-npmplus-edge.timer                                                                                                 loaded    active     waiting   Daily homelab app-level backup timer for npmplus-edge
  homelab-appbackup-postgres-batch.timer                                                                                               loaded    active     waiting   Daily Homelab Postgres app batch backup timer
  homelab-appbackup-sqlite-batch.timer                                                                                                 loaded    active     waiting   Daily Homelab SQLite app batch backup timer
  homelab-auto-backup.timer                                                                                                            loaded    active     waiting   Run homelab automatic full backup weekly
  homelab-backup-coverage-matrix.timer                                                                                                 loaded    active     waiting   Homelab backup coverage matrix timer
  homelab-backup-freshness.timer                                                                                                       loaded    active     waiting   Run homelab backup freshness collector daily
  homelab-backup-healthcheck.timer                                                                                                     loaded    active     waiting   Run homelab backup healthcheck daily
  homelab-backup-sla-health.timer                                                                                                      loaded    active     waiting   Homelab backup freshness SLA health timer
  homelab-beszel-backup.timer                                                                                                          loaded    active     waiting   Daily Beszel encrypted two-cloud backup
  homelab-blackbox-exporter-backup.timer                                                                                               loaded    active     waiting   Daily Blackbox Exporter encrypted two-cloud backup
  homelab-cluster-admin-incident-engine-backup.timer                                                                                   loaded    active     waiting   Daily Cluster Admin Incident Engine two-cloud backup
  homelab-crypto-portfolio-backup.timer                                                                                                loaded    active     waiting   Daily Gram Crypto portfolio backup
  homelab-emergency-backup.timer                                                                                                       loaded    active     waiting   Daily homelab emergency/config backup timer
  homelab-extended-appbackup.timer                                                                                                     loaded    active     waiting   Homelab extended P1/P2 appbackup timer
  homelab-gitea-secondary-backup.timer                                                                                                 loaded    active     waiting   Daily verified Gitea backup to pve03
  homelab-legacy-monitoring-backups.timer                                                                                              loaded    active     waiting   Daily legacy monitoring applications two-cloud backup batch
  homelab-mail-cloud-critical-upload.timer                                                                                             loaded    active     waiting   Daily Homelab Mail Cloud critical/small baseline upload
  homelab-mail-cloud-edge-vm-upload.timer                                                                                              loaded    active     waiting   Weekly Homelab Mail Cloud edge-vm large chunked upload
  homelab-mail-cloud-nextcloud-vm-upload.timer                                                                                         loaded    active     waiting   Run Homelab Mail Cloud Nextcloud VM upload daily
  homelab-mail-cloud-restore-drill.timer                                                                                               loaded    active     waiting   Weekly Homelab Mail Cloud restore drill
  homelab-router-running-config-mail-cloud.timer                                                                                       loaded    active     waiting   Run Homelab router running-config Mail-cloud backup daily
  homelab-verified-backup-health.timer                                                                                                 loaded    active     waiting   Homelab verified backup health evidence timer
  homelab-vm-backup-health.timer                                                                                                       loaded    active     waiting   Homelab VM backup health timer
  homelab-vm-backup-policy.timer                                                                                                       loaded    active     waiting   Homelab VM backup policy and feasibility health timer
  homelab-vm-full-backup.timer                                                                                                         loaded    active     waiting   Daily full backup timer for VM170 and VM171
  homelab-vm-local-dumps-2cloud.timer                                                                                                  loaded    active     waiting   Homelab VM/CT existing local dumps two-cloud verifier timer
  homelab-vm-local-dumps-retention.timer                                                                                               loaded    active     waiting   Homelab VM local dumps cloud retention timer
  memos-backup.timer                                                                                                                   loaded    active     waiting   Run Memos data backup daily
  memos-offhost-sync.timer                                                                                                             loaded    active     waiting   Run Memos off-host backup sync daily
  skladchik-reports-monitor-backup-restore-check.timer                                                                                 loaded    active     waiting   Weekly restore-check of Skladchik reports monitor cloud backups
  skladchik-reports-monitor-backup.timer                                                                                               loaded    active     waiting   Daily Skladchik reports monitor backup to encrypted clouds
Wed 2026-07-22 00:00:00 MSK               6h Tue 2026-07-21 00:00:14 MSK       17h ago dpkg-db-backup.timer                                   dpkg-db-backup.service
Wed 2026-07-22 00:09:43 MSK               6h Tue 2026-07-21 00:09:32 MSK       17h ago homelab-backup-freshness.timer                         homelab-backup-freshness.service
Wed 2026-07-22 02:15:00 MSK               8h Tue 2026-07-21 02:15:05 MSK       15h ago homelab-vm-full-backup.timer                           homelab-vm-full-backup.service
Wed 2026-07-22 03:10:02 MSK               9h Tue 2026-07-21 03:13:14 MSK       14h ago homelab-emergency-backup.timer                         homelab-emergency-backup.service
Wed 2026-07-22 03:35:00 MSK               9h Tue 2026-07-21 03:35:14 MSK       14h ago filebrowser-backup.timer                               filebrowser-backup.service
Wed 2026-07-22 03:55:02 MSK              10h Tue 2026-07-21 03:51:25 MSK       13h ago homelab-appbackup-dockge-stacks.timer                  homelab-appbackup-dockge-stacks.service
Wed 2026-07-22 04:17:29 MSK              10h Tue 2026-07-21 04:19:17 MSK       13h ago homelab-appbackup-npmplus-edge.timer                   homelab-appbackup-npmplus-edge.service
Wed 2026-07-22 04:26:54 MSK              10h Tue 2026-07-21 04:28:14 MSK       13h ago homelab-gitea-secondary-backup.timer                   homelab-gitea-secondary-backup.service
Wed 2026-07-22 04:38:04 MSK              10h Tue 2026-07-21 04:34:48 MSK       13h ago skladchik-reports-monitor-backup.timer                 skladchik-reports-monitor-backup.service
Wed 2026-07-22 05:01:44 MSK              11h Tue 2026-07-21 04:57:14 MSK       12h ago homelab-appbackup-sqlite-batch.timer                   homelab-appbackup-sqlite-batch.service
Wed 2026-07-22 05:23:56 MSK              11h Tue 2026-07-21 05:17:57 MSK       12h ago homelab-router-running-config-mail-cloud.timer         homelab-router-running-config-mail-cloud.service
Wed 2026-07-22 05:27:13 MSK              11h Tue 2026-07-21 05:29:05 MSK       12h ago homelab-appbackup-postgres-batch.timer                 homelab-appbackup-postgres-batch.service
Wed 2026-07-22 05:47:19 MSK              12h Tue 2026-07-21 05:46:14 MSK       11h ago homelab-appbackup-mariadb-batch.timer                  homelab-appbackup-mariadb-batch.service
Wed 2026-07-22 06:01:19 MSK              12h Tue 2026-07-21 05:57:30 MSK       11h ago homelab-extended-appbackup.timer                       homelab-extended-appbackup.service
Wed 2026-07-22 06:27:43 MSK              12h Tue 2026-07-21 06:32:25 MSK       11h ago homelab-crypto-portfolio-backup.timer                  homelab-crypto-portfolio-backup.service
Wed 2026-07-22 06:38:51 MSK              12h Tue 2026-07-21 06:43:54 MSK       10h ago homelab-verified-backup-health.timer                   homelab-verified-backup-health.service
Wed 2026-07-22 06:39:22 MSK              12h Tue 2026-07-21 06:40:25 MSK       10h ago homelab-alertmanager-backup.timer                      homelab-alertmanager-backup.service
Wed 2026-07-22 06:44:16 MSK              13h Tue 2026-07-21 06:40:25 MSK       10h ago homelab-mail-cloud-critical-upload.timer               homelab-mail-cloud-critical-upload.service
Wed 2026-07-22 06:52:06 MSK              13h Tue 2026-07-21 06:47:47 MSK       10h ago homelab-beszel-backup.timer                            homelab-beszel-backup.service
Wed 2026-07-22 06:58:58 MSK              13h Tue 2026-07-21 06:57:30 MSK       10h ago homelab-blackbox-exporter-backup.timer                 homelab-blackbox-exporter-backup.service
Wed 2026-07-22 07:02:27 MSK              13h Tue 2026-07-21 07:05:30 MSK       10h ago homelab-backup-healthcheck.timer                       homelab-backup-healthcheck.service
Wed 2026-07-22 07:07:14 MSK              13h Tue 2026-07-21 07:18:55 MSK       10h ago homelab-legacy-monitoring-backups.timer                homelab-legacy-monitoring-backups.service
Wed 2026-07-22 07:35:06 MSK              13h Tue 2026-07-21 07:36:59 MSK       10h ago homelab-vm-backup-health.timer                         homelab-vm-backup-health.service
Wed 2026-07-22 07:38:21 MSK              13h Tue 2026-07-21 07:36:14 MSK       10h ago homelab-cluster-admin-incident-engine-backup.timer     homelab-cluster-admin-incident-engine-backup.service
Wed 2026-07-22 07:41:49 MSK              14h Tue 2026-07-21 07:53:41 MSK        9h ago homelab-mail-cloud-nextcloud-vm-upload.timer           homelab-mail-cloud-nextcloud-vm-upload.service
Wed 2026-07-22 08:10:37 MSK              14h Tue 2026-07-21 08:07:25 MSK        9h ago homelab-backup-sla-health.timer                        homelab-backup-sla-health.service
Wed 2026-07-22 08:37:02 MSK              14h Tue 2026-07-21 08:30:07 MSK        9h ago homelab-vm-backup-policy.timer                         homelab-vm-backup-policy.service
Wed 2026-07-22 08:40:13 MSK              15h Tue 2026-07-21 08:50:14 MSK        8h ago homelab-backup-coverage-matrix.timer                   homelab-backup-coverage-matrix.service
Wed 2026-07-22 08:53:36 MSK              15h Tue 2026-07-21 08:41:14 MSK        8h ago memos-backup.timer                                     memos-backup.service
Sun 2026-07-26 03:34:47 MSK           4 days Sun 2026-07-19 03:45:37 MSK    2 days ago homelab-auto-backup.timer                              homelab-auto-backup.service
Sun 2026-07-26 05:17:43 MSK           4 days Sun 2026-07-19 05:11:25 MSK    2 days ago skladchik-reports-monitor-backup-restore-check.timer   skladchik-reports-monitor-backup-restore-check.service
Sun 2026-07-26 09:32:36 MSK           4 days Sun 2026-07-19 09:32:29 MSK    2 days ago homelab-mail-cloud-edge-vm-upload.timer                homelab-mail-cloud-edge-vm-upload.service
Sun 2026-07-26 10:29:15 MSK           4 days Sun 2026-07-19 10:06:07 MSK    2 days ago homelab-vm-local-dumps-2cloud.timer                    homelab-vm-local-dumps-2cloud.service
Mon 2026-07-27 07:34:51 MSK           5 days Mon 2026-07-20 07:25:14 MSK 1 day 10h ago homelab-mail-cloud-restore-drill.timer                 homelab-mail-cloud-restore-drill.service
Jul 19 20:28:30 pve01 homelab-backup-matrix-refresh[4070223]: MATRIX_STATE_STATUS=FAIL
Jul 19 20:28:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 20:28:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 20:33:30 pve01 homelab-backup-matrix-refresh[4092638]: MATRIX_STATE_STATUS=FAIL
Jul 19 20:33:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 20:33:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 20:38:36 pve01 homelab-backup-matrix-refresh[4106577]: MATRIX_STATE_STATUS=FAIL
Jul 19 20:38:36 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 20:38:36 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 20:43:56 pve01 homelab-backup-matrix-refresh[4109241]: MATRIX_STATE_STATUS=FAIL
Jul 19 20:43:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 20:43:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 20:49:15 pve01 homelab-backup-matrix-refresh[4131689]: MATRIX_STATE_STATUS=FAIL
Jul 19 20:49:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 20:49:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 20:54:15 pve01 homelab-backup-matrix-refresh[4145509]: MATRIX_STATE_STATUS=FAIL
Jul 19 20:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 20:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 20:59:15 pve01 homelab-backup-matrix-refresh[4159482]: MATRIX_STATE_STATUS=FAIL
Jul 19 20:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 20:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:04:15 pve01 homelab-backup-matrix-refresh[4181782]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:09:15 pve01 homelab-backup-matrix-refresh[1993]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:14:15 pve01 homelab-backup-matrix-refresh[16108]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:19:15 pve01 homelab-backup-matrix-refresh[38546]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:24:15 pve01 homelab-backup-matrix-refresh[52499]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:29:15 pve01 homelab-backup-matrix-refresh[66476]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:34:15 pve01 homelab-backup-matrix-refresh[88886]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:39:15 pve01 homelab-backup-matrix-refresh[102884]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:44:15 pve01 homelab-backup-matrix-refresh[116812]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:49:26 pve01 homelab-backup-matrix-refresh[139246]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:49:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:49:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:54:30 pve01 homelab-backup-matrix-refresh[153291]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 21:59:30 pve01 homelab-backup-matrix-refresh[167280]: MATRIX_STATE_STATUS=FAIL
Jul 19 21:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 21:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:04:30 pve01 homelab-backup-matrix-refresh[189663]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:09:30 pve01 homelab-backup-matrix-refresh[192354]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:14:30 pve01 homelab-backup-matrix-refresh[206349]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:19:30 pve01 homelab-backup-matrix-refresh[228779]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:24:30 pve01 homelab-backup-matrix-refresh[242779]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:29:30 pve01 homelab-backup-matrix-refresh[256597]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:34:30 pve01 homelab-backup-matrix-refresh[279054]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:34:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:34:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:39:30 pve01 homelab-backup-matrix-refresh[293022]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:44:30 pve01 homelab-backup-matrix-refresh[307038]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:49:46 pve01 homelab-backup-matrix-refresh[330036]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:49:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:49:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 22:55:15 pve01 homelab-backup-matrix-refresh[345990]: MATRIX_STATE_STATUS=FAIL
Jul 19 22:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 22:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:00:16 pve01 homelab-backup-matrix-refresh[361536]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:05:15 pve01 homelab-backup-matrix-refresh[385062]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:10:15 pve01 homelab-backup-matrix-refresh[399332]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:15:16 pve01 homelab-backup-matrix-refresh[403605]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:20:15 pve01 homelab-backup-matrix-refresh[427739]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:25:15 pve01 homelab-backup-matrix-refresh[443219]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:30:16 pve01 homelab-backup-matrix-refresh[458795]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:35:15 pve01 homelab-backup-matrix-refresh[482814]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:40:15 pve01 homelab-backup-matrix-refresh[498238]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:45:16 pve01 homelab-backup-matrix-refresh[513836]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:50:26 pve01 homelab-backup-matrix-refresh[538010]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 19 23:55:30 pve01 homelab-backup-matrix-refresh[553603]: MATRIX_STATE_STATUS=FAIL
Jul 19 23:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 19 23:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:00:16 pve01 homelab-backup-matrix-refresh[569004]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:05:16 pve01 homelab-backup-matrix-refresh[593337]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:05:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:05:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:10:16 pve01 homelab-backup-matrix-refresh[609133]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:10:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:10:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:12:16 pve01 homelab-backup-matrix-refresh[610747]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:12:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:12:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:17:16 pve01 homelab-backup-matrix-refresh[635186]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:17:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:17:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:22:17 pve01 homelab-backup-matrix-refresh[650655]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:22:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:22:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:27:26 pve01 homelab-backup-matrix-refresh[666442]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:27:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:27:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:32:30 pve01 homelab-backup-matrix-refresh[690423]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:37:30 pve01 homelab-backup-matrix-refresh[706247]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:42:30 pve01 homelab-backup-matrix-refresh[721727]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:47:30 pve01 homelab-backup-matrix-refresh[745645]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:52:30 pve01 homelab-backup-matrix-refresh[761096]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 00:57:30 pve01 homelab-backup-matrix-refresh[776579]: MATRIX_STATE_STATUS=FAIL
Jul 20 00:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 00:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:02:30 pve01 homelab-backup-matrix-refresh[797728]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:07:30 pve01 homelab-backup-matrix-refresh[804631]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:12:30 pve01 homelab-backup-matrix-refresh[820114]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:17:30 pve01 homelab-backup-matrix-refresh[844105]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:22:30 pve01 homelab-backup-matrix-refresh[859499]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:27:30 pve01 homelab-backup-matrix-refresh[875036]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:32:30 pve01 homelab-backup-matrix-refresh[899054]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:37:30 pve01 homelab-backup-matrix-refresh[914581]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:42:30 pve01 homelab-backup-matrix-refresh[930107]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:47:30 pve01 homelab-backup-matrix-refresh[954079]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:52:56 pve01 homelab-backup-matrix-refresh[969872]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 01:58:08 pve01 homelab-backup-matrix-refresh[985563]: MATRIX_STATE_STATUS=FAIL
Jul 20 01:58:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 01:58:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:03:15 pve01 homelab-backup-matrix-refresh[1009625]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:08:15 pve01 homelab-backup-matrix-refresh[1025172]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:13:15 pve01 homelab-backup-matrix-refresh[1040687]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:13:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:13:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:18:15 pve01 homelab-backup-matrix-refresh[1064737]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:18:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:18:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:23:15 pve01 homelab-backup-matrix-refresh[1080263]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:23:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:23:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:28:16 pve01 homelab-backup-matrix-refresh[1084502]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:28:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:28:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:33:15 pve01 homelab-backup-matrix-refresh[1108551]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:38:15 pve01 homelab-backup-matrix-refresh[1124188]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:43:15 pve01 homelab-backup-matrix-refresh[1139799]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:48:15 pve01 homelab-backup-matrix-refresh[1163841]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:53:15 pve01 homelab-backup-matrix-refresh[1179474]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:53:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:53:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 02:58:16 pve01 homelab-backup-matrix-refresh[1195011]: MATRIX_STATE_STATUS=FAIL
Jul 20 02:58:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 02:58:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:03:16 pve01 homelab-backup-matrix-refresh[1219515]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:03:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:03:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:08:17 pve01 homelab-backup-matrix-refresh[1235611]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:08:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:08:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:09:51 pve01 homelab-backup-matrix-refresh[1237636]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:09:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:09:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:13:42 pve01 homelab-backup-matrix-refresh[1252592]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:13:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:13:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:19:06 pve01 homelab-backup-matrix-refresh[1276835]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:19:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:19:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:24:07 pve01 homelab-backup-matrix-refresh[1292418]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:29:15 pve01 homelab-backup-matrix-refresh[1307915]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:30:00 pve01 pvescheduler[1308929]: INFO: starting new backup job: vzdump 110 111 112 113 130 150 190 --prune-backups 'keep-daily=4' --mode snapshot --mailnotification failure --storage local --compress zstd --quiet 1
Jul 20 03:34:15 pve01 homelab-backup-matrix-refresh[1332038]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:35:08 pve01 homelab-backup-matrix-refresh[1333178]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:35:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:35:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:40:15 pve01 homelab-backup-matrix-refresh[1348754]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:45:16 pve01 homelab-backup-matrix-refresh[1364258]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:50:16 pve01 homelab-backup-matrix-refresh[1388202]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:52:56 pve01 homelab-backup-matrix-refresh[1390999]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:52:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 03:58:15 pve01 homelab-backup-matrix-refresh[1406759]: MATRIX_STATE_STATUS=FAIL
Jul 20 03:58:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 03:58:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:03:15 pve01 homelab-backup-matrix-refresh[1430825]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:03:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:08:15 pve01 homelab-backup-matrix-refresh[1446346]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:08:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:10:15 pve01 homelab-backup-matrix-refresh[1448294]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:14:16 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 20 04:15:26 pve01 homelab-backup-matrix-refresh[1464728]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:15:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:15:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:20:01 pve01 homelab-backup-matrix-refresh[1488281]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:20:01 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:20:01 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:25:12 pve01 homelab-backup-matrix-refresh[1503745]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:25:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:25:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:25:47 pve01 homelab-backup-matrix-refresh[1505286]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:25:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:25:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:31:12 pve01 homelab-backup-matrix-refresh[1529373]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:31:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:31:12 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:36:15 pve01 homelab-backup-matrix-refresh[1544983]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:41:15 pve01 homelab-backup-matrix-refresh[1560445]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:46:15 pve01 homelab-backup-matrix-refresh[1584307]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:51:15 pve01 homelab-backup-matrix-refresh[1599773]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 04:56:15 pve01 homelab-backup-matrix-refresh[1615263]: MATRIX_STATE_STATUS=FAIL
Jul 20 04:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 04:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:01:16 pve01 homelab-backup-matrix-refresh[1640460]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:01:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:01:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:03:43 pve01 homelab-backup-matrix-refresh[1644969]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:03:43 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:03:43 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:08:48 pve01 homelab-backup-matrix-refresh[1660506]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:08:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:08:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:13:56 pve01 homelab-backup-matrix-refresh[1675931]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:13:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:13:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:19:07 pve01 homelab-backup-matrix-refresh[1700241]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:19:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:19:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:24:07 pve01 homelab-backup-matrix-refresh[1715665]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:24:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:27:56 pve01 homelab-backup-matrix-refresh[1733709]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:27:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:27:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:33:15 pve01 homelab-backup-matrix-refresh[1757822]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:33:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:38:15 pve01 homelab-backup-matrix-refresh[1773361]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:38:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:43:15 pve01 homelab-backup-matrix-refresh[1788819]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:43:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:48:15 pve01 homelab-backup-matrix-refresh[1812619]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:48:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:49:50 pve01 homelab-backup-matrix-refresh[1814730]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:49:50 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:49:50 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 05:54:56 pve01 homelab-backup-matrix-refresh[1830204]: MATRIX_STATE_STATUS=FAIL
Jul 20 05:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 05:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:00:16 pve01 homelab-backup-matrix-refresh[1845897]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:05:15 pve01 homelab-backup-matrix-refresh[1872116]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:10:15 pve01 homelab-backup-matrix-refresh[1893683]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:13:16 pve01 homelab-backup-matrix-refresh[1900739]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:13:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:13:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:18:18 pve01 homelab-backup-matrix-refresh[1924679]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:18:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:18:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:23:26 pve01 homelab-backup-matrix-refresh[1940188]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:23:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:23:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:28:26 pve01 homelab-backup-matrix-refresh[1956242]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:28:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:28:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:33:26 pve01 homelab-backup-matrix-refresh[1980145]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:33:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:33:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:35:21 pve01 homelab-backup-matrix-refresh[1984457]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:35:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:35:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:40:26 pve01 homelab-backup-matrix-refresh[2000528]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:45:30 pve01 homelab-backup-matrix-refresh[2016958]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:50:30 pve01 homelab-backup-matrix-refresh[2040093]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 06:52:20 pve01 homelab-mail-cloud-critical-upload[2039699]: REMOTE_FAIL=0
Jul 20 06:55:30 pve01 homelab-backup-matrix-refresh[2056859]: MATRIX_STATE_STATUS=FAIL
Jul 20 06:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 06:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:00:30 pve01 homelab-backup-matrix-refresh[2073835]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:05:30 pve01 homelab-backup-matrix-refresh[2097030]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:08:29 pve01 homelab-backup-matrix-refresh[2111023]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:08:29 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:08:29 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:13:30 pve01 homelab-backup-matrix-refresh[2123578]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:13:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:13:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:18:30 pve01 homelab-backup-matrix-refresh[2140101]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:18:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:18:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:23:30 pve01 homelab-backup-matrix-refresh[2158207]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:23:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:23:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:25:15 pve01 homelab-backup-matrix-refresh[2171082]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:30:16 pve01 homelab-backup-matrix-refresh[2186510]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:35:15 pve01 homelab-backup-matrix-refresh[2210494]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:40:21 pve01 homelab-backup-matrix-refresh[2226665]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:40:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:40:21 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:41:30 pve01 homelab-backup-matrix-refresh[2227922]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:46:32 pve01 homelab-backup-matrix-refresh[2251943]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:46:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:46:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:51:56 pve01 homelab-backup-matrix-refresh[2267857]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:51:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:51:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 07:57:08 pve01 homelab-backup-matrix-refresh[2284427]: MATRIX_STATE_STATUS=FAIL
Jul 20 07:57:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 07:57:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:02:16 pve01 homelab-backup-matrix-refresh[2297063]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:02:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:02:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:07:15 pve01 homelab-backup-matrix-refresh[2316882]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:10:25 pve01 homelab-overall-health[2331217]: STATUS=WARN TS=2026-07-20T05:10:25Z TYPE=overall-health BAD=5 backup-framework.txt=OK drift-check.txt=WARN service-registry.txt=OK dependency-map.txt=OK golden-state.txt=OK cluster-passport.txt=WARN final-readiness.txt=WARN safe-autoheal.txt=OK kuma-monitor-policy.txt=OK backup-sla.txt=OK backup-coverage-matrix.txt=OK extended-appbackup.txt=OK residual-review.txt=OK forum-snuffleupagus.txt=OK incident-journal.txt=OK duty-admin-v2.txt=OK node-loss-readiness.txt=OK node-loss-runbooks.txt=OK power-loss-readiness.txt=OK power-loss-runbook.txt=OK ungated-health-backlog.txt=OK pve03-root-cleanup.txt=OK pve03-staging-retention.txt=OK pve03-staging-retention-proof.txt=OK pve03-staging-retention-cleanup.txt=OK pve03-failed-unit-cleanup.txt=OK remote-git-sops-age-readiness.txt=OK remote-git-sops-age-policy.txt=OK desired-state-remote-target-trace.txt=OK desired-state-remote-target.txt=OK desired-state.txt=OK runbooks.txt=OK alerting.txt=OK secret-exposure.txt=OK capacity-risk.txt=OK vm-local-dumps-retention.txt=OK vm-local-dumps-cloud.txt=OK vm-backup-policy.txt=OK vm170-vm171-full-strategy.txt=OK external-probe-vps.txt=OK netbird-vps-trust.txt=OK netbird-vps-trust-closure.txt=OK external-probe-runner-decision.txt=OK netbird-vps-trust-clean-seal.txt=OK live-tail-audit.txt=OK vm-backup.txt=OK cluster-admin-observer.txt=OK cluster-admin-restricted-probes.txt=WARN cluster-admin-full-observer.txt=WARN cluster-admin-vm-mail-cloud-backup.txt=OK cluster-admin-webpanel-sync.txt=OK cluster-admin-webpanel.txt=OK
Jul 20 08:10:27 pve01 homelab-backup-matrix-refresh[2331605]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:10:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:10:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:15:30 pve01 homelab-backup-matrix-refresh[2347983]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:20:30 pve01 homelab-backup-matrix-refresh[2371091]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:25:30 pve01 homelab-backup-matrix-refresh[2386601]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:30:22 pve01 homelab-backup-matrix-refresh[2403498]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:30:22 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:30:22 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:35:26 pve01 homelab-backup-matrix-refresh[2426661]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:35:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:35:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:36:16 pve01 homelab-backup-matrix-refresh[2427543]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:36:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:36:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:41:18 pve01 homelab-backup-matrix-refresh[2443027]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:41:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:41:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:46:26 pve01 homelab-backup-matrix-refresh[2467111]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:46:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:46:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:47:15 pve01 homelab-backup-matrix-refresh[2468144]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:47:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:47:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:52:26 pve01 homelab-backup-matrix-refresh[2484169]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:52:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:52:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 08:57:30 pve01 homelab-backup-matrix-refresh[2499762]: MATRIX_STATE_STATUS=FAIL
Jul 20 08:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 08:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:02:30 pve01 homelab-backup-matrix-refresh[2523793]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:07:30 pve01 homelab-backup-matrix-refresh[2539245]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:07:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:12:30 pve01 homelab-backup-matrix-refresh[2554758]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:12:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:17:30 pve01 homelab-backup-matrix-refresh[2567370]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:17:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:22:30 pve01 homelab-backup-matrix-refresh[2583117]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:22:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:27:30 pve01 homelab-backup-matrix-refresh[2598614]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:27:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:32:30 pve01 homelab-backup-matrix-refresh[2622549]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:32:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:37:30 pve01 homelab-backup-matrix-refresh[2638103]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:37:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:42:30 pve01 homelab-backup-matrix-refresh[2653680]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:42:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:47:30 pve01 homelab-backup-matrix-refresh[2677702]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:47:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:52:30 pve01 homelab-backup-matrix-refresh[2693207]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:52:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 09:57:30 pve01 homelab-backup-matrix-refresh[2708789]: MATRIX_STATE_STATUS=FAIL
Jul 20 09:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 09:57:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:02:30 pve01 homelab-backup-matrix-refresh[2732734]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:02:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:04:07 pve01 homelab-backup-matrix-refresh[2734125]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:04:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:04:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:09:07 pve01 homelab-backup-matrix-refresh[2749537]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:09:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:09:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:14:07 pve01 homelab-backup-matrix-refresh[2765091]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:14:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:14:07 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:19:15 pve01 homelab-backup-matrix-refresh[2789006]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:24:15 pve01 homelab-backup-matrix-refresh[2804462]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:29:15 pve01 homelab-backup-matrix-refresh[2819907]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:34:15 pve01 homelab-backup-matrix-refresh[2843900]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:34:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:39:15 pve01 homelab-backup-matrix-refresh[2859366]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:39:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:44:16 pve01 homelab-backup-matrix-refresh[2863481]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:44:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:44:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:49:15 pve01 homelab-backup-matrix-refresh[2887434]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:54:15 pve01 homelab-backup-matrix-refresh[2902771]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 10:59:15 pve01 homelab-backup-matrix-refresh[2918259]: MATRIX_STATE_STATUS=FAIL
Jul 20 10:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 10:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:04:15 pve01 homelab-backup-matrix-refresh[2942234]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:09:15 pve01 homelab-backup-matrix-refresh[2957726]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:14:15 pve01 homelab-backup-matrix-refresh[2973175]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:19:15 pve01 homelab-backup-matrix-refresh[2997114]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:19:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:24:15 pve01 homelab-backup-matrix-refresh[3012539]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:24:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:29:15 pve01 homelab-backup-matrix-refresh[3028098]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:29:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:34:26 pve01 homelab-backup-matrix-refresh[3052177]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:34:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:34:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:39:30 pve01 homelab-backup-matrix-refresh[3067593]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:39:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:44:30 pve01 homelab-backup-matrix-refresh[3083139]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:44:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:49:30 pve01 homelab-backup-matrix-refresh[3107024]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:49:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:49:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:54:30 pve01 homelab-backup-matrix-refresh[3122507]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:54:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 11:59:30 pve01 homelab-backup-matrix-refresh[3137990]: MATRIX_STATE_STATUS=FAIL
Jul 20 11:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 11:59:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:04:30 pve01 homelab-backup-matrix-refresh[3150598]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:04:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:09:30 pve01 homelab-backup-matrix-refresh[3166051]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:09:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:14:30 pve01 homelab-backup-matrix-refresh[3181454]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:14:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:19:30 pve01 homelab-backup-matrix-refresh[3205349]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:19:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:24:30 pve01 homelab-backup-matrix-refresh[3220920]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:24:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:29:30 pve01 homelab-backup-matrix-refresh[3236455]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:29:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:34:56 pve01 homelab-backup-matrix-refresh[3260683]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:34:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:34:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:40:08 pve01 homelab-backup-matrix-refresh[3276379]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:40:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:40:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:45:09 pve01 homelab-backup-matrix-refresh[3297387]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:45:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:45:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:50:15 pve01 homelab-backup-matrix-refresh[3315975]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 12:55:15 pve01 homelab-backup-matrix-refresh[3331521]: MATRIX_STATE_STATUS=FAIL
Jul 20 12:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 12:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:00:15 pve01 homelab-backup-matrix-refresh[3347832]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:05:15 pve01 homelab-backup-matrix-refresh[3370899]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:10:15 pve01 homelab-backup-matrix-refresh[3386358]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:15:16 pve01 homelab-backup-matrix-refresh[3401894]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:20:15 pve01 homelab-backup-matrix-refresh[3425830]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:25:15 pve01 homelab-backup-matrix-refresh[3429905]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:30:15 pve01 homelab-backup-matrix-refresh[3446239]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:30:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:30:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:35:15 pve01 homelab-backup-matrix-refresh[3469372]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:40:15 pve01 homelab-backup-matrix-refresh[3484818]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:45:16 pve01 homelab-backup-matrix-refresh[3500319]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:45:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:50:26 pve01 homelab-backup-matrix-refresh[3524417]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:50:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 13:55:30 pve01 homelab-backup-matrix-refresh[3539958]: MATRIX_STATE_STATUS=FAIL
Jul 20 13:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 13:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:00:30 pve01 homelab-backup-matrix-refresh[3556311]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:05:30 pve01 homelab-backup-matrix-refresh[3579422]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:05:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:10:30 pve01 homelab-backup-matrix-refresh[3594909]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:10:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:10:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:15:30 pve01 homelab-backup-matrix-refresh[3618728]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:15:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:20:30 pve01 homelab-backup-matrix-refresh[3634183]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:20:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:25:30 pve01 homelab-backup-matrix-refresh[3649674]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:25:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:30:30 pve01 homelab-backup-matrix-refresh[3673515]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:30:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:30:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:35:30 pve01 homelab-backup-matrix-refresh[3689095]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:35:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:35:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:40:31 pve01 homelab-backup-matrix-refresh[3698580]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:45:30 pve01 homelab-backup-matrix-refresh[3709574]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:45:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:50:30 pve01 homelab-backup-matrix-refresh[3732500]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 14:55:30 pve01 homelab-backup-matrix-refresh[3747946]: MATRIX_STATE_STATUS=FAIL
Jul 20 14:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 14:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:00:30 pve01 homelab-backup-matrix-refresh[3771782]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:00:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:05:48 pve01 homelab-backup-matrix-refresh[3787552]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:05:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:05:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:11:06 pve01 homelab-backup-matrix-refresh[3803194]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:11:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:11:06 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:16:16 pve01 homelab-backup-matrix-refresh[3819582]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:16:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:16:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:21:15 pve01 homelab-backup-matrix-refresh[3842674]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:26:15 pve01 homelab-backup-matrix-refresh[3858134]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:26:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:26:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:31:15 pve01 homelab-backup-matrix-refresh[3874469]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:36:15 pve01 homelab-backup-matrix-refresh[3897548]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:41:15 pve01 homelab-backup-matrix-refresh[3912924]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:46:16 pve01 homelab-backup-matrix-refresh[3917964]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:46:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:46:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:51:15 pve01 homelab-backup-matrix-refresh[3941009]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:51:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 15:56:15 pve01 homelab-backup-matrix-refresh[3956553]: MATRIX_STATE_STATUS=FAIL
Jul 20 15:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 15:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:01:15 pve01 homelab-backup-matrix-refresh[3972944]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:01:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:01:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:06:15 pve01 homelab-backup-matrix-refresh[3996047]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:06:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:06:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:11:15 pve01 homelab-backup-matrix-refresh[4011569]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:11:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:16:15 pve01 homelab-backup-matrix-refresh[4035432]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:16:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:16:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:21:15 pve01 homelab-backup-matrix-refresh[4050883]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:21:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:26:16 pve01 homelab-backup-matrix-refresh[4066337]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:26:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:26:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:31:18 pve01 homelab-backup-matrix-refresh[4090328]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:31:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:31:18 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:36:26 pve01 homelab-backup-matrix-refresh[4106097]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:36:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:36:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:41:30 pve01 homelab-backup-matrix-refresh[4121654]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:46:30 pve01 homelab-backup-matrix-refresh[4145630]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:51:30 pve01 homelab-backup-matrix-refresh[4161061]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 16:56:31 pve01 homelab-backup-matrix-refresh[4176634]: MATRIX_STATE_STATUS=FAIL
Jul 20 16:56:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 16:56:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:01:30 pve01 homelab-backup-matrix-refresh[6881]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:06:30 pve01 homelab-backup-matrix-refresh[22575]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:11:30 pve01 homelab-backup-matrix-refresh[38083]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:16:31 pve01 homelab-backup-matrix-refresh[59658]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:21:30 pve01 homelab-backup-matrix-refresh[66183]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:26:30 pve01 homelab-backup-matrix-refresh[81598]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:31:31 pve01 homelab-backup-matrix-refresh[105617]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:36:30 pve01 homelab-backup-matrix-refresh[121433]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:36:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:36:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:41:30 pve01 homelab-backup-matrix-refresh[136937]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:41:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:46:30 pve01 homelab-backup-matrix-refresh[160897]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:46:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:51:30 pve01 homelab-backup-matrix-refresh[176488]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:51:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 17:56:30 pve01 homelab-backup-matrix-refresh[191911]: MATRIX_STATE_STATUS=FAIL
Jul 20 17:56:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 17:56:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:01:30 pve01 homelab-backup-matrix-refresh[215914]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:01:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:06:30 pve01 homelab-backup-matrix-refresh[231465]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:06:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:11:30 pve01 homelab-backup-matrix-refresh[247009]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:11:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:16:31 pve01 homelab-backup-matrix-refresh[270971]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:16:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:21:30 pve01 homelab-backup-matrix-refresh[286516]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:21:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:26:30 pve01 homelab-backup-matrix-refresh[290662]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:26:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:31:30 pve01 homelab-backup-matrix-refresh[314846]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:31:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:36:46 pve01 homelab-backup-matrix-refresh[330621]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:36:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:36:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:41:48 pve01 homelab-backup-matrix-refresh[346168]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:41:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:41:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:46:58 pve01 homelab-backup-matrix-refresh[370235]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:46:58 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:46:58 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:52:08 pve01 homelab-backup-matrix-refresh[385867]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:52:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:52:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 18:57:15 pve01 homelab-backup-matrix-refresh[401443]: MATRIX_STATE_STATUS=FAIL
Jul 20 18:57:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 18:57:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:02:15 pve01 homelab-backup-matrix-refresh[425446]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:02:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:02:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:07:15 pve01 homelab-backup-matrix-refresh[440954]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:07:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:12:15 pve01 homelab-backup-matrix-refresh[456367]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:12:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:12:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:17:15 pve01 homelab-backup-matrix-refresh[480328]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:17:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:17:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:22:15 pve01 homelab-backup-matrix-refresh[495871]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:22:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:22:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:27:15 pve01 homelab-backup-matrix-refresh[511358]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:27:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:27:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:28:51 pve01 homelab-backup-matrix-refresh[512760]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:28:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:28:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:28:59 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 20 19:34:08 pve01 homelab-backup-matrix-refresh[537480]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:34:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:34:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:39:09 pve01 homelab-backup-matrix-refresh[553115]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:39:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:39:09 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:44:15 pve01 homelab-backup-matrix-refresh[568668]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:44:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:49:15 pve01 homelab-backup-matrix-refresh[592723]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:49:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:54:15 pve01 homelab-backup-matrix-refresh[608272]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:54:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 19:59:15 pve01 homelab-backup-matrix-refresh[623767]: MATRIX_STATE_STATUS=FAIL
Jul 20 19:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 19:59:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:04:15 pve01 homelab-backup-matrix-refresh[647775]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:04:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:09:15 pve01 homelab-backup-matrix-refresh[663153]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:09:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:14:15 pve01 homelab-backup-matrix-refresh[667315]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:14:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:19:26 pve01 homelab-backup-matrix-refresh[691333]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:19:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:19:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:24:31 pve01 homelab-backup-matrix-refresh[706901]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:24:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:24:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:29:31 pve01 homelab-backup-matrix-refresh[722368]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:29:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:29:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:34:31 pve01 homelab-backup-matrix-refresh[746391]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:34:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:34:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:39:31 pve01 homelab-backup-matrix-refresh[761801]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:39:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:39:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:44:31 pve01 homelab-backup-matrix-refresh[777273]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:44:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:44:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:49:31 pve01 homelab-backup-matrix-refresh[801193]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:49:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:49:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 20:54:56 pve01 homelab-backup-matrix-refresh[816954]: MATRIX_STATE_STATUS=FAIL
Jul 20 20:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 20:54:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:00:16 pve01 homelab-backup-matrix-refresh[832684]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:05:15 pve01 homelab-backup-matrix-refresh[856679]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:10:15 pve01 homelab-backup-matrix-refresh[872167]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:15:16 pve01 homelab-backup-matrix-refresh[887656]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:20:15 pve01 homelab-backup-matrix-refresh[911605]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:25:16 pve01 homelab-backup-matrix-refresh[915711]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:25:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:25:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:30:16 pve01 homelab-backup-matrix-refresh[931433]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:30:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:35:16 pve01 homelab-backup-matrix-refresh[955099]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:40:26 pve01 homelab-backup-matrix-refresh[970563]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:40:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:45:28 pve01 homelab-backup-matrix-refresh[986962]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:45:28 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:45:28 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:50:30 pve01 homelab-backup-matrix-refresh[1009969]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:50:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 21:55:30 pve01 homelab-backup-matrix-refresh[1025505]: MATRIX_STATE_STATUS=FAIL
Jul 20 21:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 21:55:30 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:00:31 pve01 homelab-backup-matrix-refresh[1041885]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:05:31 pve01 homelab-backup-matrix-refresh[1065096]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:10:31 pve01 homelab-backup-matrix-refresh[1080543]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:15:31 pve01 homelab-backup-matrix-refresh[1096922]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:15:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:15:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:20:31 pve01 homelab-backup-matrix-refresh[1120354]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:20:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:20:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:25:31 pve01 homelab-backup-matrix-refresh[1135777]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:25:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:25:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:30:31 pve01 homelab-backup-matrix-refresh[1152095]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:30:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:30:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:35:31 pve01 homelab-backup-matrix-refresh[1175243]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:35:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:35:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:40:31 pve01 homelab-backup-matrix-refresh[1179415]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:40:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:45:31 pve01 homelab-backup-matrix-refresh[1195757]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:45:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:45:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:50:31 pve01 homelab-backup-matrix-refresh[1218911]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:50:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:50:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 22:55:31 pve01 homelab-backup-matrix-refresh[1234331]: MATRIX_STATE_STATUS=FAIL
Jul 20 22:55:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 22:55:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:00:31 pve01 homelab-backup-matrix-refresh[1258173]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:00:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:05:31 pve01 homelab-backup-matrix-refresh[1273772]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:10:31 pve01 homelab-backup-matrix-refresh[1289212]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:15:32 pve01 homelab-backup-matrix-refresh[1313105]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:15:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:15:32 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:20:48 pve01 homelab-backup-matrix-refresh[1328836]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:20:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:20:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:25:49 pve01 homelab-backup-matrix-refresh[1344339]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:25:49 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:25:49 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:31:15 pve01 homelab-backup-matrix-refresh[1360964]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:31:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:36:15 pve01 homelab-backup-matrix-refresh[1384068]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:36:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:41:15 pve01 homelab-backup-matrix-refresh[1399485]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:46:15 pve01 homelab-backup-matrix-refresh[1423238]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:46:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:51:16 pve01 homelab-backup-matrix-refresh[1427834]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:51:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:51:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 20 23:56:15 pve01 homelab-backup-matrix-refresh[1443462]: MATRIX_STATE_STATUS=FAIL
Jul 20 23:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 20 23:56:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:00:16 pve01 homelab-backup-matrix-refresh[1458364]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:00:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:05:26 pve01 homelab-backup-matrix-refresh[1482896]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:05:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:05:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:09:33 pve01 homelab-backup-matrix-refresh[1498014]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:09:33 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:09:33 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:09:34 pve01 homelab-safe-exec[1498016]: homelab-vm160-remote-backup.service     loaded failed failed VM160 full-image backup streamed from pve02 to pve03
Jul 21 00:10:55 pve01 homelab-safe-exec[1499369]: homelab-backup-matrix-refresh.service   loaded failed failed Generate unified homelab backup matrix state
Jul 21 00:10:55 pve01 homelab-safe-exec[1499369]: homelab-vm160-remote-backup.service     loaded failed failed VM160 full-image backup streamed from pve02 to pve03
Jul 21 00:14:48 pve01 homelab-backup-matrix-refresh[1513957]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:14:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:14:48 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:20:00 pve01 homelab-backup-matrix-refresh[1538116]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:20:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:20:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:25:04 pve01 homelab-backup-matrix-refresh[1553654]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:25:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:25:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:26:08 pve01 homelab-safe-exec[1554627]: homelab-backup-matrix-refresh.service   loaded failed failed Generate unified homelab backup matrix state
Jul 21 00:26:08 pve01 homelab-safe-exec[1554627]: homelab-vm160-remote-backup.service     loaded failed failed VM160 full-image backup streamed from pve02 to pve03
Jul 21 00:30:05 pve01 homelab-backup-matrix-refresh[1569592]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:30:05 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:30:05 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:35:16 pve01 homelab-backup-matrix-refresh[1594059]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:35:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:40:15 pve01 homelab-backup-matrix-refresh[1609512]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:42:17 pve01 homelab-safe-exec[1611283]: homelab-backup-matrix-refresh.service   loaded failed failed Generate unified homelab backup matrix state
Jul 21 00:42:17 pve01 homelab-safe-exec[1611283]: homelab-vm160-remote-backup.service     loaded failed failed VM160 full-image backup streamed from pve02 to pve03
Jul 21 00:45:15 pve01 homelab-backup-matrix-refresh[1633504]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:45:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:45:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:50:15 pve01 homelab-backup-matrix-refresh[1637691]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:55:15 pve01 homelab-backup-matrix-refresh[1653171]: MATRIX_STATE_STATUS=FAIL
Jul 21 00:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 00:55:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 00:58:17 pve01 homelab-safe-exec[1667105]: homelab-backup-matrix-refresh.service   loaded failed failed Generate unified homelab backup matrix state
Jul 21 00:58:17 pve01 homelab-safe-exec[1667105]: homelab-vm160-remote-backup.service     loaded failed failed VM160 full-image backup streamed from pve02 to pve03
Jul 21 01:00:15 pve01 homelab-backup-matrix-refresh[1669525]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:00:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:05:15 pve01 homelab-backup-matrix-refresh[1692700]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:05:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:10:15 pve01 homelab-backup-matrix-refresh[1708083]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:10:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:15:16 pve01 homelab-backup-matrix-refresh[1723575]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:15:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:15:18 pve01 homelab-safe-exec[1723574]: homelab-vm160-remote-backup.service     loaded failed failed VM160 full-image backup streamed from pve02 to pve03
Jul 21 01:20:15 pve01 homelab-backup-matrix-refresh[1747690]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:20:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:25:15 pve01 homelab-backup-matrix-refresh[1763279]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:25:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:30:15 pve01 homelab-backup-matrix-refresh[1778809]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:30:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:30:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:32:09 pve01 homelab-safe-exec[1789005]: homelab-backup-matrix-refresh.service   loaded failed failed Generate unified homelab backup matrix state
Jul 21 01:32:09 pve01 homelab-safe-exec[1789005]: homelab-vm160-remote-backup.service     loaded failed failed VM160 full-image backup streamed from pve02 to pve03
Jul 21 01:35:15 pve01 homelab-backup-matrix-refresh[1802971]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:40:15 pve01 homelab-backup-matrix-refresh[1818394]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:40:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:45:15 pve01 homelab-backup-matrix-refresh[1834787]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:45:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:45:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:48:17 pve01 homelab-safe-exec[1844870]: homelab-backup-matrix-refresh.service   loaded failed failed Generate unified homelab backup matrix state
Jul 21 01:48:17 pve01 homelab-safe-exec[1844870]: homelab-vm160-remote-backup.service     loaded failed failed VM160 full-image backup streamed from pve02 to pve03
Jul 21 01:50:15 pve01 homelab-backup-matrix-refresh[1857955]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:50:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 01:55:26 pve01 homelab-backup-matrix-refresh[1873552]: MATRIX_STATE_STATUS=FAIL
Jul 21 01:55:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 01:55:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 02:00:26 pve01 homelab-backup-matrix-refresh[1878517]: MATRIX_STATE_STATUS=FAIL
Jul 21 02:00:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:00:26 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 02:03:27 pve01 homelab-safe-exec[1900187]: homelab-backup-matrix-refresh.service   loaded failed failed Generate unified homelab backup matrix state
Jul 21 02:03:27 pve01 homelab-safe-exec[1900187]: homelab-vm160-remote-backup.service     loaded failed failed VM160 full-image backup streamed from pve02 to pve03
Jul 21 02:05:31 pve01 homelab-backup-matrix-refresh[1901804]: MATRIX_STATE_STATUS=FAIL
Jul 21 02:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:05:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 02:10:31 pve01 homelab-backup-matrix-refresh[1917432]: MATRIX_STATE_STATUS=FAIL
Jul 21 02:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:10:31 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 02:14:04 pve01 homelab-backup-matrix-refresh[1932401]: MATRIX_STATE_STATUS=FAIL
Jul 21 02:14:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:14:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 02:14:13 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:14:13 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 21 02:20:13 pve01 homelab-backup-matrix-refresh[1957976]: MATRIX_STATE_STATUS=FAIL
Jul 21 02:20:13 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:20:13 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 02:20:21 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:20:21 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 21 02:33:04 pve01 homelab-backup-matrix-refresh[2011530]: MATRIX_STATE_STATUS=FAIL
Jul 21 02:33:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:33:04 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 02:39:14 pve01 systemd[1]: homelab-vm160-remote-backup.service: Failed with result 'exit-code'.
Jul 21 03:09:44 pve01 homelab-backup-matrix-refresh[2136275]: MATRIX_STATE_STATUS=FAIL
Jul 21 03:09:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 03:09:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 03:13:44 pve01 homelab-backup-matrix-refresh[2138763]: MATRIX_STATE_STATUS=FAIL
Jul 21 03:13:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 03:13:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 03:30:02 pve01 pvescheduler[2189056]: INFO: starting new backup job: vzdump 110 111 112 113 130 150 190 --compress zstd --quiet 1 --mode snapshot --prune-backups 'keep-daily=4' --mailnotification failure --storage local
Jul 21 03:35:15 pve01 homelab-backup-matrix-refresh[2211416]: MATRIX_STATE_STATUS=FAIL
Jul 21 03:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 03:35:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 03:51:56 pve01 homelab-backup-matrix-refresh[2262529]: MATRIX_STATE_STATUS=FAIL
Jul 21 03:51:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 03:51:56 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 04:10:08 pve01 homelab-backup-matrix-refresh[2324930]: MATRIX_STATE_STATUS=FAIL
Jul 21 04:10:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 04:10:08 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 04:19:51 pve01 homelab-backup-matrix-refresh[2361594]: MATRIX_STATE_STATUS=FAIL
Jul 21 04:19:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 04:19:51 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 04:35:23 pve01 homelab-backup-matrix-refresh[2412406]: MATRIX_STATE_STATUS=FAIL
Jul 21 04:35:23 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 04:35:23 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 05:00:46 pve01 homelab-backup-matrix-refresh[2482684]: MATRIX_STATE_STATUS=FAIL
Jul 21 05:00:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 05:00:46 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 05:32:33 pve01 appbackupctl[2593535]: Failed to lsf with 2 errors: last error was: error in ListJSON: file error 400 ()
Jul 21 05:32:39 pve01 homelab-backup-matrix-refresh[2593863]: MATRIX_STATE_STATUS=FAIL
Jul 21 05:32:39 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 05:32:39 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 05:46:47 pve01 homelab-backup-matrix-refresh[2643920]: MATRIX_STATE_STATUS=FAIL
Jul 21 05:46:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 05:46:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 06:06:47 pve01 homelab-backup-matrix-refresh[2708415]: MATRIX_STATE_STATUS=FAIL
Jul 21 06:06:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 06:06:47 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 06:42:44 pve01 homelab-mail-cloud-critical-upload[2820985]: REMOTE_FAIL=0
Jul 21 06:45:17 pve01 homelab-backup-matrix-refresh[2839359]: MATRIX_STATE_STATUS=FAIL
Jul 21 06:45:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 06:45:17 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 07:05:44 pve01 homelab-backup-matrix-refresh[2900406]: MATRIX_STATE_STATUS=FAIL
Jul 21 07:05:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 07:05:44 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 07:37:00 pve01 homelab-backup-matrix-refresh[3011793]: MATRIX_STATE_STATUS=FAIL
Jul 21 07:37:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 07:37:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 08:01:15 pve01 homelab-overall-health[3098671]: STATUS=WARN TS=2026-07-21T05:01:14Z TYPE=overall-health BAD=7 backup-framework.txt=OK drift-check.txt=WARN service-registry.txt=OK dependency-map.txt=OK golden-state.txt=OK cluster-passport.txt=WARN final-readiness.txt=WARN safe-autoheal.txt=OK kuma-monitor-policy.txt=OK backup-sla.txt=WARN backup-coverage-matrix.txt=OK extended-appbackup.txt=OK residual-review.txt=OK forum-snuffleupagus.txt=OK incident-journal.txt=OK duty-admin-v2.txt=OK node-loss-readiness.txt=OK node-loss-runbooks.txt=OK power-loss-readiness.txt=OK power-loss-runbook.txt=OK ungated-health-backlog.txt=OK pve03-root-cleanup.txt=OK pve03-staging-retention.txt=OK pve03-staging-retention-proof.txt=OK pve03-staging-retention-cleanup.txt=OK pve03-failed-unit-cleanup.txt=OK remote-git-sops-age-readiness.txt=OK remote-git-sops-age-policy.txt=OK desired-state-remote-target-trace.txt=OK desired-state-remote-target.txt=OK desired-state.txt=OK runbooks.txt=OK alerting.txt=OK secret-exposure.txt=OK capacity-risk.txt=OK vm-local-dumps-retention.txt=OK vm-local-dumps-cloud.txt=OK vm-backup-policy.txt=OK vm170-vm171-full-strategy.txt=OK external-probe-vps.txt=OK netbird-vps-trust.txt=OK netbird-vps-trust-closure.txt=OK external-probe-runner-decision.txt=OK netbird-vps-trust-clean-seal.txt=OK live-tail-audit.txt=OK vm-backup.txt=OK cluster-admin-observer.txt=WARN cluster-admin-restricted-probes.txt=WARN cluster-admin-full-observer.txt=WARN cluster-admin-vm-mail-cloud-backup.txt=OK cluster-admin-webpanel-sync.txt=OK cluster-admin-webpanel.txt=OK
Jul 21 08:07:27 pve01 homelab-backup-matrix-refresh[3122695]: MATRIX_STATE_STATUS=FAIL
Jul 21 08:07:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 08:07:27 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 08:30:14 pve01 homelab-backup-matrix-refresh[3200030]: MATRIX_STATE_STATUS=FAIL
Jul 21 08:30:14 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 08:30:14 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 08:41:15 pve01 homelab-backup-matrix-refresh[3239614]: MATRIX_STATE_STATUS=FAIL
Jul 21 08:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 08:41:15 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 08:50:16 pve01 homelab-backup-matrix-refresh[3277954]: MATRIX_STATE_STATUS=FAIL
Jul 21 08:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 08:50:16 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
Jul 21 10:02:42 pve01 homelab-backup-matrix-refresh[3525445]: MATRIX_STATE_STATUS=FAIL
Jul 21 10:02:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
Jul 21 10:02:42 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
CRITICAL_FIRST_DIAGNOSTIC=COMPLETE

OUTPUT_END
CHAT_OUTPUT_END
