diff --git a/runtime/history/BACKUP-OFFHOST-DELTA-212.json b/runtime/history/BACKUP-OFFHOST-DELTA-212.json new file mode 100644 index 0000000..c17a06c --- /dev/null +++ b/runtime/history/BACKUP-OFFHOST-DELTA-212.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "channel": "homelab-runtime", + "command_id": "BACKUP-OFFHOST-DELTA-212", + "status": "OK", + "rc": 0, + "host": "pve01", + "mode": "read-only", + "component": "cluster-audit", + "started_at_utc": "2026-07-22T15:00:05Z", + "finished_at_utc": "2026-07-22T15:00:05Z", + "reference_register_checked": true, + "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", + "error_register_checked": true, + "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", + "command_sha256": "cd7bfba0a3e05e136f85c1362ba4b34c9e142ef49f4621874743aebf2b665eee", + "duplicate_failed_command_blocked": false, + "block_reason": null, + "execution_started": true, + "changes_made": false, + "sanitized": true, + "secrets_included": false, + "private_addresses_included": false, + "raw_evidence_retained_locally": true, + "raw_evidence_sha256": "10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682", + "sanitized_output_sha256": "10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682", + "output_truncated_in_json": false, + "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", + "output": "validation=PASS\n=== TARGET_UNITS ===\nResult=exit-code\nExecMainStatus=1\nId=homelab-backup-matrix-refresh.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\n\nResult=exit-code\nExecMainStatus=1\nId=homelab-vm160-remote-backup.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\n\nResult=exit-code\nExecMainStatus=1\nId=homelab-cluster-admin-pull.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\n=== TARGET_FAILURES ===\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nhomelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01\nhomelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\n=== HEALTH_MARKERS ===\nFILE=/var/lib/homelab-health/backup-framework.txt MTIME=2026-07-22 10:11:15.588809944 +0300 SIZE=803\nSTATUS=OK\nTS=2026-07-22T07:11:15Z\nTYPE=backup-matrix-generated\nFILE=/var/lib/homelab-health/backup-sla.txt MTIME=2026-07-22 17:45:04.576070554 +0300 SIZE=154\nSTATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nFILE=/var/lib/homelab-health/backup-coverage-matrix.txt MTIME=2026-07-22 10:11:15.236509358 +0300 SIZE=808\nSTATUS=OK\nTS=2026-07-22T07:11:15Z\nTYPE=backup-matrix-generated\nFILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt MTIME=2026-07-22 10:11:15.694007276 +0300 SIZE=800\nSTATUS=OK\nTS=2026-07-22T07:11:15Z\nTYPE=backup-matrix-generated\nFILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt MTIME=2026-07-09 08:07:47.213452046 +0300 SIZE=217\nSTATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\nFILE=/var/lib/homelab-health/vm-backup.txt MTIME=2026-07-22 07:35:14.956502473 +0300 SIZE=170\nSTATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active\n=== LOCAL_ARCHIVES ===\nARCHIVE=vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400\nARCHIVE=vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720\nARCHIVE=vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330\nARCHIVE=vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500\nARCHIVE=vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730\nARCHIVE=vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440\nARCHIVE=vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290\nARCHIVE=vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120\nARCHIVE=vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020\nARCHIVE=vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470\nARCHIVE=vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950\nARCHIVE=vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120\n=== RECENT_ERRORS ===\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nCHANGES_MADE=NO\nREFERENCE_CHECK=OK\nERROR_REGISTER_CHECK=OK\nRESULT=OK\nBACKUP_OFFHOST_DELTA=COMPLETE\n" +} diff --git a/runtime/history/BACKUP-OFFHOST-DELTA-212.txt b/runtime/history/BACKUP-OFFHOST-DELTA-212.txt new file mode 100644 index 0000000..b1ecf49 --- /dev/null +++ b/runtime/history/BACKUP-OFFHOST-DELTA-212.txt @@ -0,0 +1,123 @@ +CHAT_OUTPUT_BEGIN +COMMAND_ID=BACKUP-OFFHOST-DELTA-212 +STATUS=OK +RC=0 +HOST=pve01 +MODE=read-only +COMPONENT=cluster-audit +REFERENCE_REGISTER_CHECK=OK +REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e +ERROR_REGISTER_CHECK=OK +ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 +COMMAND_SHA256=cd7bfba0a3e05e136f85c1362ba4b34c9e142ef49f4621874743aebf2b665eee +DUPLICATE_FAILED_COMMAND_BLOCKED=false +EXECUTION_STARTED=true +CHANGES_MADE=false +SANITIZED=yes +SECRETS_INCLUDED=no +PRIVATE_ADDRESSES_INCLUDED=no +RAW_EVIDENCE_SHA256=10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682 +SANITIZED_OUTPUT_SHA256=10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682 +OUTPUT_BEGIN +validation=PASS +=== TARGET_UNITS === +Result=exit-code +ExecMainStatus=1 +Id=homelab-backup-matrix-refresh.service +LoadState=loaded +ActiveState=failed +SubState=failed + +Result=exit-code +ExecMainStatus=1 +Id=homelab-vm160-remote-backup.service +LoadState=loaded +ActiveState=failed +SubState=failed + +Result=exit-code +ExecMainStatus=1 +Id=homelab-cluster-admin-pull.service +LoadState=loaded +ActiveState=failed +SubState=failed +=== TARGET_FAILURES === +homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state +homelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01 +homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 +=== HEALTH_MARKERS === +FILE=/var/lib/homelab-health/backup-framework.txt MTIME=2026-07-22 10:11:15.588809944 +0300 SIZE=803 +STATUS=OK +TS=2026-07-22T07:11:15Z +TYPE=backup-matrix-generated +FILE=/var/lib/homelab-health/backup-sla.txt MTIME=2026-07-22 17:45:04.576070554 +0300 SIZE=154 +STATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md +FILE=/var/lib/homelab-health/backup-coverage-matrix.txt MTIME=2026-07-22 10:11:15.236509358 +0300 SIZE=808 +STATUS=OK +TS=2026-07-22T07:11:15Z +TYPE=backup-matrix-generated +FILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt MTIME=2026-07-22 10:11:15.694007276 +0300 SIZE=800 +STATUS=OK +TS=2026-07-22T07:11:15Z +TYPE=backup-matrix-generated +FILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt MTIME=2026-07-09 08:07:47.213452046 +0300 SIZE=217 +STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md +FILE=/var/lib/homelab-health/vm-backup.txt MTIME=2026-07-22 07:35:14.956502473 +0300 SIZE=170 +STATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active +=== LOCAL_ARCHIVES === +ARCHIVE=vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400 +ARCHIVE=vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720 +ARCHIVE=vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330 +ARCHIVE=vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500 +ARCHIVE=vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730 +ARCHIVE=vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440 +ARCHIVE=vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290 +ARCHIVE=vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120 +ARCHIVE=vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020 +ARCHIVE=vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470 +ARCHIVE=vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950 +ARCHIVE=vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120 +=== RECENT_ERRORS === +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +CHANGES_MADE=NO +REFERENCE_CHECK=OK +ERROR_REGISTER_CHECK=OK +RESULT=OK +BACKUP_OFFHOST_DELTA=COMPLETE + +OUTPUT_END +CHAT_OUTPUT_END diff --git a/runtime/latest.json b/runtime/latest.json index b40175f..c17a06c 100644 --- a/runtime/latest.json +++ b/runtime/latest.json @@ -1,19 +1,19 @@ { "schema_version": 1, "channel": "homelab-runtime", - "command_id": "BACKUP-OFFHOST-CURRENT-STATE-212", + "command_id": "BACKUP-OFFHOST-DELTA-212", "status": "OK", "rc": 0, "host": "pve01", "mode": "read-only", - "component": "backup-audit", - "started_at_utc": "2026-07-22T14:53:51Z", - "finished_at_utc": "2026-07-22T14:53:55Z", + "component": "cluster-audit", + "started_at_utc": "2026-07-22T15:00:05Z", + "finished_at_utc": "2026-07-22T15:00:05Z", "reference_register_checked": true, "reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e", "error_register_checked": true, "error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752", - "command_sha256": "5b9f985ab6daceaad18e56d2da3dde0d2dba7eefdfe7f75e83eee6bef0a0ad8a", + "command_sha256": "cd7bfba0a3e05e136f85c1362ba4b34c9e142ef49f4621874743aebf2b665eee", "duplicate_failed_command_blocked": false, "block_reason": null, "execution_started": true, @@ -22,9 +22,9 @@ "secrets_included": false, "private_addresses_included": false, "raw_evidence_retained_locally": true, - "raw_evidence_sha256": "9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2", - "sanitized_output_sha256": "9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2", + "raw_evidence_sha256": "10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682", + "sanitized_output_sha256": "10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682", "output_truncated_in_json": false, "full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt", - "output": "validation=PASS\nPREVIOUS_RESULT_204=VERIFIED_OK\nHISTORICAL_BACKUP_BASELINE_122=VERIFIED\nREFERENCE_CHECK=OK\nERROR_REGISTER_CHECK=OK\nAUDIT_GENERATED_AT=2026-07-22T14:53:51Z\n=== CURRENT_REFERENCE_STATE ===\n{\"status\":\"AUTO_REFRESH_READY\",\"next_action\":null,\"updated_at\":\"2026-07-22T14:22:14+00:00\"}\n=== CLUSTER_NODES ===\n{\"node\":\"pve01\",\"status\":\"online\",\"uptime\":540902}\n{\"node\":\"pve02\",\"status\":\"online\",\"uptime\":540921}\n{\"node\":\"pve03\",\"status\":\"online\",\"uptime\":540774}\n=== PVE_BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784766600,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== STORAGE_STATUS ===\nName Type Status Total (KiB) Used (KiB) Available (KiB) %\nlocal dir active 102626232 42074800 55292168 41.00%\nlocal-lvm lvmthin active 848797696 305906689 542891006 36.04%\n========== NODE=pve01 ==========\nSSH_STATUS=OK\nHOST=pve01\nUTC_NOW=2026-07-22T14:53:53Z\n=== FAILED_UNITS ===\nFAILED_UNIT_COUNT=3\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nhomelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01\nhomelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\n=== TARGET_UNITS ===\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-backup-matrix-refresh.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 10:11:15 MSK\nResult=success\nId=homelab-backup-matrix-refresh.timer\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=disabled\nActiveEnterTimestamp=\nInactiveEnterTimestamp=\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-vm160-remote-backup.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Tue 2026-07-21 02:39:14 MSK\nResult=success\nId=homelab-vm160-remote-backup.timer\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=disabled\nActiveEnterTimestamp=\nInactiveEnterTimestamp=\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-cluster-admin-pull ; argv[]=/usr/local/sbin/homelab-cluster-admin-pull ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-cluster-admin-pull.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Tue 2026-07-21 02:37:04 MSK\nResult=success\nId=homelab-cluster-admin-pull.timer\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=disabled\nActiveEnterTimestamp=\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-vm-local-dumps-2cloud.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Sun 2026-07-19 10:13:40 MSK\nResult=success\nId=homelab-vm-local-dumps-2cloud.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-critical-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-critical-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-critical-upload.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 06:46:47 MSK\nResult=success\nId=homelab-mail-cloud-critical-upload.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-edge-vm-upload.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Sun 2026-07-19 09:41:56 MSK\nResult=success\nId=homelab-mail-cloud-edge-vm-upload.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-nextcloud-vm-upload.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 07:45:01 MSK\nResult=success\nId=homelab-mail-cloud-nextcloud-vm-upload.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-restore-drill.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Mon 2026-07-20 07:25:14 MSK\nResult=success\nId=homelab-mail-cloud-restore-drill.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-health-evidence ; argv[]=/usr/local/sbin/homelab-backup-health-evidence ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-verified-backup-health.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 06:40:29 MSK\nResult=success\nId=homelab-verified-backup-health.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-sla-health ; argv[]=/usr/local/sbin/homelab-backup-sla-health ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-backup-sla-health.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 08:10:42 MSK\nResult=success\nId=homelab-backup-sla-health.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\n=== RELEVANT_TIMERS ===\nThu 2026-07-23 06:38:07 MSK 12h Wed 2026-07-22 06:39:05 MSK 11h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nThu 2026-07-23 06:44:08 MSK 12h Wed 2026-07-22 06:44:25 MSK 11h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nThu 2026-07-23 07:58:35 MSK 14h Wed 2026-07-22 07:42:14 MSK 10h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nThu 2026-07-23 08:12:51 MSK 14h Wed 2026-07-22 08:10:41 MSK 9h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nSun 2026-07-26 09:46:53 MSK 3 days Sun 2026-07-19 09:32:29 MSK 3 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:23:39 MSK 3 days Sun 2026-07-19 10:06:07 MSK 3 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:27 MSK 4 days Mon 2026-07-20 07:25:14 MSK 2 days ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== TARGET_SCRIPT_STATE ===\nFILE=/usr/local/sbin/homelab-backup-matrix-refresh TYPE=regular file MODE=755 OWNER=root:root SIZE=8714 MTIME=2026-07-10 13:02:04.846868245 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-backup-matrix-refresh:FAIL\nFILE=/usr/local/sbin/homelab-vm160-remote-backup TYPE=regular file MODE=755 OWNER=root:root SIZE=3140 MTIME=2026-07-18 01:38:18.170629793 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-vm160-remote-backup:OK\nFILE=/usr/local/sbin/homelab-cluster-admin-pull TYPE=regular file MODE=755 OWNER=root:root SIZE=2501 MTIME=2026-07-13 01:15:06.431144817 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-cluster-admin-pull:OK\nFILE=/usr/local/sbin/homelab-vm-local-dumps-2cloud TYPE=regular file MODE=700 OWNER=root:root SIZE=7272 MTIME=2026-07-06 18:49:49.740645050 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-vm-local-dumps-2cloud:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-critical-upload TYPE=regular file MODE=700 OWNER=root:root SIZE=7836 MTIME=2026-07-17 14:03:50.373696143 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-critical-upload:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=12899 MTIME=2026-07-13 21:27:22.164609073 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=6542 MTIME=2026-06-30 15:00:33.588620116 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-restore-drill TYPE=regular file MODE=700 OWNER=root:root SIZE=7995 MTIME=2026-07-10 04:14:10.054957455 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-restore-drill:OK\n=== CURRENT_BACKUP_HEALTH ===\nHEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt\nHEALTH_FILE=/var/lib/homelab-health/backup-framework.txt\nHEALTH_MTIME=2026-07-22 10:11:15.588809944 +0300 HEALTH_SIZE=803\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/backup-sla.txt\nHEALTH_MTIME=2026-07-22 17:45:04.576070554 +0300 HEALTH_SIZE=154\nSTATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nHEALTH_FILE=/var/lib/homelab-health/backup-coverage-matrix.txt\nHEALTH_MTIME=2026-07-22 10:11:15.236509358 +0300 HEALTH_SIZE=808\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt\nHEALTH_MTIME=2026-07-22 10:11:15.694007276 +0300 HEALTH_SIZE=800\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-retention.txt\nHEALTH_MTIME=2026-07-22 10:11:15.711074323 +0300 HEALTH_SIZE=826\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/vm-backup.txt\nHEALTH_MTIME=2026-07-22 07:35:14.956502473 +0300 HEALTH_SIZE=170\nSTATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active\nHEALTH_FILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\nHEALTH_MTIME=2026-07-09 08:07:47.213452046 +0300 HEALTH_SIZE=217\nSTATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\nHEALTH_FILE=/var/lib/homelab-health/edge-vm-offhost-freshness.txt\nHEALTH_MTIME=2026-07-22 04:46:23.189237991 +0300 HEALTH_SIZE=409\nSTATUS=OK TS=2026-07-22T01:39:30Z TYPE=edge-vm-offhost-freshness BAD=0 VMID=130 SOURCE_NODE=pve03 DEST_NODE=pve02 SRC=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst DEST=/mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 SHA256=a8e5179614e02e6933617adb30cb95a35a1b5d17b30c9f00a2626cfa06b14710 COPY_STATUS=copied ZSTD_RC=0 VMA_RC=0 KEEP=4\nHEALTH_FILE=/var/lib/homelab-health/mail-cloud-restore-drill.txt\nHEALTH_MTIME=2026-07-22 10:11:15.491775958 +0300 HEALTH_SIZE=830\nSTATUS=STALE\n=== RCLONE_REMOTE_NAMES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== ACTIVE_VZDUMP ===\n1599768 bash /usr/local/sbin/homelab-chat-run BACKUP-OFFHOST-CURRENT-STATE-212 read-only backup-audit -- /usr/bin/bash -c set -eu test \"$(hostname -s)\" = \"pve01\" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W=\"$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)\" trap \"rm -rf -- \\\"$W\\\"\" EXIT BASE=\"https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history\" curl -fsSL \\ \"$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json\" \\ -o \"$W/204.json\" jq -e \" .command_id == \\\"HOMELAB-TAILS-FINAL-CLOSEOUT-204\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \\\"\\\") | contains(\\\"ALL_TAILS_CLOSED=YES\\\")) and ((.output // \\\"\\\") | contains(\\\"TAILS_FINAL_CLOSEOUT=COMPLETE\\\")) \" \"$W/204.json\" >/dev/null curl -fsSL \\ \"$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json\" \\ -o \"$W/122.json\" jq -e \" .command_id == \\\"HOMELAB-INDEPENDENT-FINAL-STATE-122\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and ((.output // \\\"\\\") | contains(\\\"HOMELAB_FINAL_STATE=BLOCKED\\\")) and ((.output // \\\"\\\") | contains(\\\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\\\")) \" \"$W/122.json\" >/dev/null echo \"PREVIOUS_RESULT_204=VERIFIED_OK\" echo \"HISTORICAL_BACKUP_BASELINE_122=VERIFIED\" echo \"REFERENCE_CHECK=OK\" echo \"ERROR_REGISTER_CHECK=OK\" echo \"AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" echo \"=== CURRENT_REFERENCE_STATE ===\" STATE=\"/etc/homelab/reference/state/current.json\" if test -s \"$STATE\" && jq -e \".\" \"$STATE\" >/dev/null; then jq -c \"{status,next_action,updated_at}\" \"$STATE\" else echo \"REFERENCE_STATE=INVALID_OR_MISSING\" fi echo \"=== CLUSTER_NODES ===\" pvesh get /nodes --output-format json | jq -c \"sort_by(.node)[] | {node,status,uptime}\" echo \"=== PVE_BACKUP_JOBS ===\" pvesh get /cluster/backup --output-format json | jq -c \"sort_by(.id)\" echo \"=== STORAGE_STATUS ===\" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo \"========== NODE=$NODE ==========\" if ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" true then echo \"SSH_STATUS=OK\" ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" \" set -u echo HOST=\\$(hostname -s) echo UTC_NOW=\\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \\ homelab-backup-matrix-refresh.service \\ homelab-backup-matrix-refresh.timer \\ homelab-vm160-remote-backup.service \\ homelab-vm160-remote-backup.timer \\ homelab-cluster-admin-pull.service \\ homelab-cluster-admin-pull.timer \\ homelab-vm-local-dumps-2cloud.service \\ homelab-vm-local-dumps-2cloud.timer \\ homelab-mail-cloud-critical-upload.service \\ homelab-mail-cloud-critical-upload.timer \\ homelab-mail-cloud-edge-vm-upload.service \\ homelab-mail-cloud-edge-vm-upload.timer \\ homelab-mail-cloud-nextcloud-vm-upload.service \\ homelab-mail-cloud-nextcloud-vm-upload.timer \\ homelab-mail-cloud-restore-drill.service \\ homelab-mail-cloud-restore-drill.timer \\ homelab-verified-backup-health.service \\ homelab-verified-backup-health.timer \\ homelab-backup-sla-health.service \\ homelab-backup-sla-health.timer do if systemctl cat \\\"\\$U\\\" >/dev/null 2>&1 then systemctl show \\\"\\$U\\\" \\ -p Id \\ -p LoadState \\ -p ActiveState \\ -p SubState \\ -p Result \\ -p ExecMainStatus \\ -p UnitFileState \\ -p ActiveEnterTimestamp \\ -p InactiveEnterTimestamp \\ -p ExecStart \\ --no-pager else echo UNIT_MISSING=\\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \\\"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\\\" || true echo === TARGET_SCRIPT_STATE === for F in \\ /usr/local/sbin/homelab-backup-matrix-refresh \\ /usr/local/sbin/homelab-vm160-remote-backup \\ /usr/local/sbin/homelab-cluster-admin-pull \\ /usr/local/sbin/homelab-vm-local-dumps-2cloud \\ /usr/local/sbin/homelab-mail-cloud-critical-upload \\ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \\\"\\$F\\\" then stat -c \\\"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\\\" \\\"\\$F\\\" if /usr/bin/bash -n \\\"\\$F\\\" 2>/dev/null then echo SCRIPT_SYNTAX=\\$F:OK else echo SCRIPT_SYNTAX=\\$F:FAIL fi else echo FILE_MISSING=\\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \\ /var/lib/homelab-health/unified-backup-matrix.txt \\ /var/lib/homelab-health/backup-framework.txt \\ /var/lib/homelab-health/backup-sla.txt \\ /var/lib/homelab-health/backup-coverage-matrix.txt \\ /var/lib/homelab-health/vm-local-dumps-cloud.txt \\ /var/lib/homelab-health/vm-local-dumps-retention.txt \\ /var/lib/homelab-health/vm-backup.txt \\ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \\ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \\ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \\\"\\$H\\\" then echo HEALTH_FILE=\\$H stat -c \\\"HEALTH_MTIME=%y HEALTH_SIZE=%s\\\" \\\"\\$H\\\" sed -n \\\"1p\\\" \\\"\\$H\\\" else echo HEALTH_FILE_MISSING=\\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \\\"[v]zdump\\\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf x \\ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf \\\"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\\\" \\ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \\ -u homelab-backup-matrix-refresh.service \\ -u homelab-vm160-remote-backup.service \\ -u homelab-cluster-admin-pull.service \\ -n 120 \\ --no-pager \\ -o short-iso \\ 2>/dev/null || true \" else echo \"SSH_STATUS=FAIL\" fi done echo \"CHANGES_MADE=NO\" echo \"SECRET_PRINTED=NO\" echo \"PRIVATE_ADDRESS_PRINTED=NO\" echo \"RESULT=OK\" echo \"BACKUP_OFFHOST_CURRENT_STATE=COMPLETE\" \n1599834 timeout --foreground --preserve-status 300 /usr/bin/bash -c set -eu test \"$(hostname -s)\" = \"pve01\" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W=\"$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)\" trap \"rm -rf -- \\\"$W\\\"\" EXIT BASE=\"https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history\" curl -fsSL \\ \"$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json\" \\ -o \"$W/204.json\" jq -e \" .command_id == \\\"HOMELAB-TAILS-FINAL-CLOSEOUT-204\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \\\"\\\") | contains(\\\"ALL_TAILS_CLOSED=YES\\\")) and ((.output // \\\"\\\") | contains(\\\"TAILS_FINAL_CLOSEOUT=COMPLETE\\\")) \" \"$W/204.json\" >/dev/null curl -fsSL \\ \"$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json\" \\ -o \"$W/122.json\" jq -e \" .command_id == \\\"HOMELAB-INDEPENDENT-FINAL-STATE-122\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and ((.output // \\\"\\\") | contains(\\\"HOMELAB_FINAL_STATE=BLOCKED\\\")) and ((.output // \\\"\\\") | contains(\\\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\\\")) \" \"$W/122.json\" >/dev/null echo \"PREVIOUS_RESULT_204=VERIFIED_OK\" echo \"HISTORICAL_BACKUP_BASELINE_122=VERIFIED\" echo \"REFERENCE_CHECK=OK\" echo \"ERROR_REGISTER_CHECK=OK\" echo \"AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" echo \"=== CURRENT_REFERENCE_STATE ===\" STATE=\"/etc/homelab/reference/state/current.json\" if test -s \"$STATE\" && jq -e \".\" \"$STATE\" >/dev/null; then jq -c \"{status,next_action,updated_at}\" \"$STATE\" else echo \"REFERENCE_STATE=INVALID_OR_MISSING\" fi echo \"=== CLUSTER_NODES ===\" pvesh get /nodes --output-format json | jq -c \"sort_by(.node)[] | {node,status,uptime}\" echo \"=== PVE_BACKUP_JOBS ===\" pvesh get /cluster/backup --output-format json | jq -c \"sort_by(.id)\" echo \"=== STORAGE_STATUS ===\" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo \"========== NODE=$NODE ==========\" if ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" true then echo \"SSH_STATUS=OK\" ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" \" set -u echo HOST=\\$(hostname -s) echo UTC_NOW=\\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \\ homelab-backup-matrix-refresh.service \\ homelab-backup-matrix-refresh.timer \\ homelab-vm160-remote-backup.service \\ homelab-vm160-remote-backup.timer \\ homelab-cluster-admin-pull.service \\ homelab-cluster-admin-pull.timer \\ homelab-vm-local-dumps-2cloud.service \\ homelab-vm-local-dumps-2cloud.timer \\ homelab-mail-cloud-critical-upload.service \\ homelab-mail-cloud-critical-upload.timer \\ homelab-mail-cloud-edge-vm-upload.service \\ homelab-mail-cloud-edge-vm-upload.timer \\ homelab-mail-cloud-nextcloud-vm-upload.service \\ homelab-mail-cloud-nextcloud-vm-upload.timer \\ homelab-mail-cloud-restore-drill.service \\ homelab-mail-cloud-restore-drill.timer \\ homelab-verified-backup-health.service \\ homelab-verified-backup-health.timer \\ homelab-backup-sla-health.service \\ homelab-backup-sla-health.timer do if systemctl cat \\\"\\$U\\\" >/dev/null 2>&1 then systemctl show \\\"\\$U\\\" \\ -p Id \\ -p LoadState \\ -p ActiveState \\ -p SubState \\ -p Result \\ -p ExecMainStatus \\ -p UnitFileState \\ -p ActiveEnterTimestamp \\ -p InactiveEnterTimestamp \\ -p ExecStart \\ --no-pager else echo UNIT_MISSING=\\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \\\"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\\\" || true echo === TARGET_SCRIPT_STATE === for F in \\ /usr/local/sbin/homelab-backup-matrix-refresh \\ /usr/local/sbin/homelab-vm160-remote-backup \\ /usr/local/sbin/homelab-cluster-admin-pull \\ /usr/local/sbin/homelab-vm-local-dumps-2cloud \\ /usr/local/sbin/homelab-mail-cloud-critical-upload \\ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \\\"\\$F\\\" then stat -c \\\"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\\\" \\\"\\$F\\\" if /usr/bin/bash -n \\\"\\$F\\\" 2>/dev/null then echo SCRIPT_SYNTAX=\\$F:OK else echo SCRIPT_SYNTAX=\\$F:FAIL fi else echo FILE_MISSING=\\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \\ /var/lib/homelab-health/unified-backup-matrix.txt \\ /var/lib/homelab-health/backup-framework.txt \\ /var/lib/homelab-health/backup-sla.txt \\ /var/lib/homelab-health/backup-coverage-matrix.txt \\ /var/lib/homelab-health/vm-local-dumps-cloud.txt \\ /var/lib/homelab-health/vm-local-dumps-retention.txt \\ /var/lib/homelab-health/vm-backup.txt \\ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \\ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \\ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \\\"\\$H\\\" then echo HEALTH_FILE=\\$H stat -c \\\"HEALTH_MTIME=%y HEALTH_SIZE=%s\\\" \\\"\\$H\\\" sed -n \\\"1p\\\" \\\"\\$H\\\" else echo HEALTH_FILE_MISSING=\\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \\\"[v]zdump\\\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf x \\ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf \\\"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\\\" \\ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \\ -u homelab-backup-matrix-refresh.service \\ -u homelab-vm160-remote-backup.service \\ -u homelab-cluster-admin-pull.service \\ -n 120 \\ --no-pager \\ -o short-iso \\ 2>/dev/null || true \" else echo \"SSH_STATUS=FAIL\" fi done echo \"CHANGES_MADE=NO\" echo \"SECRET_PRINTED=NO\" echo \"PRIVATE_ADDRESS_PRINTED=NO\" echo \"RESULT=OK\" echo \"BACKUP_OFFHOST_CURRENT_STATE=COMPLETE\" \n1599836 /usr/bin/bash -c set -eu test \"$(hostname -s)\" = \"pve01\" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W=\"$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)\" trap \"rm -rf -- \\\"$W\\\"\" EXIT BASE=\"https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history\" curl -fsSL \\ \"$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json\" \\ -o \"$W/204.json\" jq -e \" .command_id == \\\"HOMELAB-TAILS-FINAL-CLOSEOUT-204\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \\\"\\\") | contains(\\\"ALL_TAILS_CLOSED=YES\\\")) and ((.output // \\\"\\\") | contains(\\\"TAILS_FINAL_CLOSEOUT=COMPLETE\\\")) \" \"$W/204.json\" >/dev/null curl -fsSL \\ \"$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json\" \\ -o \"$W/122.json\" jq -e \" .command_id == \\\"HOMELAB-INDEPENDENT-FINAL-STATE-122\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and ((.output // \\\"\\\") | contains(\\\"HOMELAB_FINAL_STATE=BLOCKED\\\")) and ((.output // \\\"\\\") | contains(\\\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\\\")) \" \"$W/122.json\" >/dev/null echo \"PREVIOUS_RESULT_204=VERIFIED_OK\" echo \"HISTORICAL_BACKUP_BASELINE_122=VERIFIED\" echo \"REFERENCE_CHECK=OK\" echo \"ERROR_REGISTER_CHECK=OK\" echo \"AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" echo \"=== CURRENT_REFERENCE_STATE ===\" STATE=\"/etc/homelab/reference/state/current.json\" if test -s \"$STATE\" && jq -e \".\" \"$STATE\" >/dev/null; then jq -c \"{status,next_action,updated_at}\" \"$STATE\" else echo \"REFERENCE_STATE=INVALID_OR_MISSING\" fi echo \"=== CLUSTER_NODES ===\" pvesh get /nodes --output-format json | jq -c \"sort_by(.node)[] | {node,status,uptime}\" echo \"=== PVE_BACKUP_JOBS ===\" pvesh get /cluster/backup --output-format json | jq -c \"sort_by(.id)\" echo \"=== STORAGE_STATUS ===\" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo \"========== NODE=$NODE ==========\" if ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" true then echo \"SSH_STATUS=OK\" ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" \" set -u echo HOST=\\$(hostname -s) echo UTC_NOW=\\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \\ homelab-backup-matrix-refresh.service \\ homelab-backup-matrix-refresh.timer \\ homelab-vm160-remote-backup.service \\ homelab-vm160-remote-backup.timer \\ homelab-cluster-admin-pull.service \\ homelab-cluster-admin-pull.timer \\ homelab-vm-local-dumps-2cloud.service \\ homelab-vm-local-dumps-2cloud.timer \\ homelab-mail-cloud-critical-upload.service \\ homelab-mail-cloud-critical-upload.timer \\ homelab-mail-cloud-edge-vm-upload.service \\ homelab-mail-cloud-edge-vm-upload.timer \\ homelab-mail-cloud-nextcloud-vm-upload.service \\ homelab-mail-cloud-nextcloud-vm-upload.timer \\ homelab-mail-cloud-restore-drill.service \\ homelab-mail-cloud-restore-drill.timer \\ homelab-verified-backup-health.service \\ homelab-verified-backup-health.timer \\ homelab-backup-sla-health.service \\ homelab-backup-sla-health.timer do if systemctl cat \\\"\\$U\\\" >/dev/null 2>&1 then systemctl show \\\"\\$U\\\" \\ -p Id \\ -p LoadState \\ -p ActiveState \\ -p SubState \\ -p Result \\ -p ExecMainStatus \\ -p UnitFileState \\ -p ActiveEnterTimestamp \\ -p InactiveEnterTimestamp \\ -p ExecStart \\ --no-pager else echo UNIT_MISSING=\\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \\\"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\\\" || true echo === TARGET_SCRIPT_STATE === for F in \\ /usr/local/sbin/homelab-backup-matrix-refresh \\ /usr/local/sbin/homelab-vm160-remote-backup \\ /usr/local/sbin/homelab-cluster-admin-pull \\ /usr/local/sbin/homelab-vm-local-dumps-2cloud \\ /usr/local/sbin/homelab-mail-cloud-critical-upload \\ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \\\"\\$F\\\" then stat -c \\\"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\\\" \\\"\\$F\\\" if /usr/bin/bash -n \\\"\\$F\\\" 2>/dev/null then echo SCRIPT_SYNTAX=\\$F:OK else echo SCRIPT_SYNTAX=\\$F:FAIL fi else echo FILE_MISSING=\\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \\ /var/lib/homelab-health/unified-backup-matrix.txt \\ /var/lib/homelab-health/backup-framework.txt \\ /var/lib/homelab-health/backup-sla.txt \\ /var/lib/homelab-health/backup-coverage-matrix.txt \\ /var/lib/homelab-health/vm-local-dumps-cloud.txt \\ /var/lib/homelab-health/vm-local-dumps-retention.txt \\ /var/lib/homelab-health/vm-backup.txt \\ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \\ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \\ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \\\"\\$H\\\" then echo HEALTH_FILE=\\$H stat -c \\\"HEALTH_MTIME=%y HEALTH_SIZE=%s\\\" \\\"\\$H\\\" sed -n \\\"1p\\\" \\\"\\$H\\\" else echo HEALTH_FILE_MISSING=\\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \\\"[v]zdump\\\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf x \\ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf \\\"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\\\" \\ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \\ -u homelab-backup-matrix-refresh.service \\ -u homelab-vm160-remote-backup.service \\ -u homelab-cluster-admin-pull.service \\ -n 120 \\ --no-pager \\ -o short-iso \\ 2>/dev/null || true \" else echo \"SSH_STATUS=FAIL\" fi done echo \"CHANGES_MADE=NO\" echo \"SECRET_PRINTED=NO\" echo \"PRIVATE_ADDRESS_PRINTED=NO\" echo \"RESULT=OK\" echo \"BACKUP_OFFHOST_CURRENT_STATE=COMPLETE\" \n1603200 ssh -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes root@pve01 set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat \"$U\" >/dev/null 2>&1 then systemctl show \"$U\" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\" if /usr/bin/bash -n \"$F\" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"$H\" then echo HEALTH_FILE=$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\" sed -n \"1p\" \"$H\" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true \n1603210 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat \"$U\" >/dev/null 2>&1 then systemctl show \"$U\" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\" if /usr/bin/bash -n \"$F\" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"$H\" then echo HEALTH_FILE=$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\" sed -n \"1p\" \"$H\" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true \n=== LOCAL_ARCHIVES ===\nARCHIVE_COUNT=12\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120\n=== TARGET_UNIT_LOGS ===\n2026-07-22T05:48:02+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T05:48:03+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T06:11:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T06:11:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T06:40:29+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T06:40:30+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T07:02:40+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T07:02:41+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T07:35:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T07:35:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:10:42+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:10:43+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:37:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:40:15+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:40:16+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:53:36+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:53:37+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T10:11:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n========== NODE=pve02 ==========\nSSH_STATUS=OK\nHOST=pve02\nUTC_NOW=2026-07-22T14:53:54Z\n=== FAILED_UNITS ===\nFAILED_UNIT_COUNT=0\n=== TARGET_UNITS ===\nUNIT_MISSING=homelab-backup-matrix-refresh.service\nUNIT_MISSING=homelab-backup-matrix-refresh.timer\nUNIT_MISSING=homelab-vm160-remote-backup.service\nUNIT_MISSING=homelab-vm160-remote-backup.timer\nUNIT_MISSING=homelab-cluster-admin-pull.service\nUNIT_MISSING=homelab-cluster-admin-pull.timer\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.service\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.timer\nUNIT_MISSING=homelab-mail-cloud-critical-upload.service\nUNIT_MISSING=homelab-mail-cloud-critical-upload.timer\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-restore-drill.service\nUNIT_MISSING=homelab-mail-cloud-restore-drill.timer\nUNIT_MISSING=homelab-verified-backup-health.service\nUNIT_MISSING=homelab-verified-backup-health.timer\nUNIT_MISSING=homelab-backup-sla-health.service\nUNIT_MISSING=homelab-backup-sla-health.timer\n=== RELEVANT_TIMERS ===\n=== TARGET_SCRIPT_STATE ===\nFILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh\nFILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup\nFILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull\nFILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill\n=== CURRENT_BACKUP_HEALTH ===\nHEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt\n=== RCLONE_REMOTE_NAMES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== ACTIVE_VZDUMP ===\n3930634 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat \"$U\" >/dev/null 2>&1 then systemctl show \"$U\" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\" if /usr/bin/bash -n \"$F\" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"$H\" then echo HEALTH_FILE=$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\" sed -n \"1p\" \"$H\" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true \n=== LOCAL_ARCHIVES ===\nARCHIVE_COUNT=10\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst SIZE=284443608 MTIME=2026-07-19T03:47:41.8525503200\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst SIZE=286538526 MTIME=2026-07-20T03:30:18.4386603270\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst SIZE=288349157 MTIME=2026-07-21T03:30:21.2480085730\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_22-03_30_01.tar.zst SIZE=290059742 MTIME=2026-07-22T03:30:22.8748248510\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst SIZE=220049041 MTIME=2026-07-19T03:47:58.9256407960\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst SIZE=220065825 MTIME=2026-07-20T03:30:30.8287264090\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst SIZE=220080970 MTIME=2026-07-21T03:30:33.3910787180\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_22-03_30_23.tar.zst SIZE=220095608 MTIME=2026-07-22T03:30:40.6809207660\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst SIZE=868048369 MTIME=2026-06-30T03:31:08.4903256780\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst SIZE=997503422 MTIME=2026-07-01T03:31:11.1700059310\n=== TARGET_UNIT_LOGS ===\n-- No entries --\n========== NODE=pve03 ==========\nSSH_STATUS=OK\nHOST=pve03\nUTC_NOW=2026-07-22T14:53:55Z\n=== FAILED_UNITS ===\nFAILED_UNIT_COUNT=1\nhomelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image\n=== TARGET_UNITS ===\nUNIT_MISSING=homelab-backup-matrix-refresh.service\nUNIT_MISSING=homelab-backup-matrix-refresh.timer\nUNIT_MISSING=homelab-vm160-remote-backup.service\nUNIT_MISSING=homelab-vm160-remote-backup.timer\nUNIT_MISSING=homelab-cluster-admin-pull.service\nUNIT_MISSING=homelab-cluster-admin-pull.timer\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.service\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.timer\nUNIT_MISSING=homelab-mail-cloud-critical-upload.service\nUNIT_MISSING=homelab-mail-cloud-critical-upload.timer\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-restore-drill.service\nUNIT_MISSING=homelab-mail-cloud-restore-drill.timer\nUNIT_MISSING=homelab-verified-backup-health.service\nUNIT_MISSING=homelab-verified-backup-health.timer\nUNIT_MISSING=homelab-backup-sla-health.service\nUNIT_MISSING=homelab-backup-sla-health.timer\n=== RELEVANT_TIMERS ===\n=== TARGET_SCRIPT_STATE ===\nFILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh\nFILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup\nFILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull\nFILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill\n=== CURRENT_BACKUP_HEALTH ===\nHEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt\n=== RCLONE_REMOTE_NAMES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== ACTIVE_VZDUMP ===\n2313989 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat \"$U\" >/dev/null 2>&1 then systemctl show \"$U\" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\" if /usr/bin/bash -n \"$F\" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"$H\" then echo HEALTH_FILE=$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\" sed -n \"1p\" \"$H\" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true \n=== LOCAL_ARCHIVES ===\nARCHIVE_COUNT=8\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst SIZE=11013723404 MTIME=2026-07-19T03:33:53.3463878040\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst SIZE=11116472459 MTIME=2026-07-20T03:33:53.2839064370\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst SIZE=11043834894 MTIME=2026-07-21T03:33:54.3725716230\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 MTIME=2026-07-22T03:33:58.8961071480\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst SIZE=838951976 MTIME=2026-07-19T03:34:33.1034859540\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst SIZE=838817925 MTIME=2026-07-20T03:34:37.1719936670\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst SIZE=1137072350 MTIME=2026-07-21T03:34:37.6276811360\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_22-03_33_59.vma.zst SIZE=1177407443 MTIME=2026-07-22T03:34:44.3412293530\n=== TARGET_UNIT_LOGS ===\n-- No entries --\nCHANGES_MADE=NO\nSECRET_PRINTED=NO\nPRIVATE_ADDRESS_PRINTED=NO\nRESULT=OK\nBACKUP_OFFHOST_CURRENT_STATE=COMPLETE\n" + "output": "validation=PASS\n=== TARGET_UNITS ===\nResult=exit-code\nExecMainStatus=1\nId=homelab-backup-matrix-refresh.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\n\nResult=exit-code\nExecMainStatus=1\nId=homelab-vm160-remote-backup.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\n\nResult=exit-code\nExecMainStatus=1\nId=homelab-cluster-admin-pull.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\n=== TARGET_FAILURES ===\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nhomelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01\nhomelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\n=== HEALTH_MARKERS ===\nFILE=/var/lib/homelab-health/backup-framework.txt MTIME=2026-07-22 10:11:15.588809944 +0300 SIZE=803\nSTATUS=OK\nTS=2026-07-22T07:11:15Z\nTYPE=backup-matrix-generated\nFILE=/var/lib/homelab-health/backup-sla.txt MTIME=2026-07-22 17:45:04.576070554 +0300 SIZE=154\nSTATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nFILE=/var/lib/homelab-health/backup-coverage-matrix.txt MTIME=2026-07-22 10:11:15.236509358 +0300 SIZE=808\nSTATUS=OK\nTS=2026-07-22T07:11:15Z\nTYPE=backup-matrix-generated\nFILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt MTIME=2026-07-22 10:11:15.694007276 +0300 SIZE=800\nSTATUS=OK\nTS=2026-07-22T07:11:15Z\nTYPE=backup-matrix-generated\nFILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt MTIME=2026-07-09 08:07:47.213452046 +0300 SIZE=217\nSTATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\nFILE=/var/lib/homelab-health/vm-backup.txt MTIME=2026-07-22 07:35:14.956502473 +0300 SIZE=170\nSTATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active\n=== LOCAL_ARCHIVES ===\nARCHIVE=vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400\nARCHIVE=vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720\nARCHIVE=vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330\nARCHIVE=vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500\nARCHIVE=vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730\nARCHIVE=vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440\nARCHIVE=vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290\nARCHIVE=vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120\nARCHIVE=vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020\nARCHIVE=vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470\nARCHIVE=vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950\nARCHIVE=vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120\n=== RECENT_ERRORS ===\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nMATRIX_STATE_STATUS=FAIL\nhomelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\nhomelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\nFailed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\nCHANGES_MADE=NO\nREFERENCE_CHECK=OK\nERROR_REGISTER_CHECK=OK\nRESULT=OK\nBACKUP_OFFHOST_DELTA=COMPLETE\n" } diff --git a/runtime/latest.txt b/runtime/latest.txt index 49a8224..b1ecf49 100644 --- a/runtime/latest.txt +++ b/runtime/latest.txt @@ -1,640 +1,123 @@ CHAT_OUTPUT_BEGIN -COMMAND_ID=BACKUP-OFFHOST-CURRENT-STATE-212 +COMMAND_ID=BACKUP-OFFHOST-DELTA-212 STATUS=OK RC=0 HOST=pve01 MODE=read-only -COMPONENT=backup-audit +COMPONENT=cluster-audit REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 -COMMAND_SHA256=5b9f985ab6daceaad18e56d2da3dde0d2dba7eefdfe7f75e83eee6bef0a0ad8a +COMMAND_SHA256=cd7bfba0a3e05e136f85c1362ba4b34c9e142ef49f4621874743aebf2b665eee DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no -RAW_EVIDENCE_SHA256=9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2 -SANITIZED_OUTPUT_SHA256=9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2 +RAW_EVIDENCE_SHA256=10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682 +SANITIZED_OUTPUT_SHA256=10222d1e1bc7b8e7635fe6fc3e47ecf85cf351b6efef5f3d843b33836e1b0682 OUTPUT_BEGIN validation=PASS -PREVIOUS_RESULT_204=VERIFIED_OK -HISTORICAL_BACKUP_BASELINE_122=VERIFIED -REFERENCE_CHECK=OK -ERROR_REGISTER_CHECK=OK -AUDIT_GENERATED_AT=2026-07-22T14:53:51Z -=== CURRENT_REFERENCE_STATE === -{"status":"AUTO_REFRESH_READY","next_action":null,"updated_at":"2026-07-22T14:22:14+00:00"} -=== CLUSTER_NODES === -{"node":"pve01","status":"online","uptime":540902} -{"node":"pve02","status":"online","uptime":540921} -{"node":"pve03","status":"online","uptime":540774} -=== PVE_BACKUP_JOBS === -[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784766600,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] -=== STORAGE_STATUS === -Name Type Status Total (KiB) Used (KiB) Available (KiB) % -local dir active 102626232 42074800 55292168 41.00% -local-lvm lvmthin active 848797696 305906689 542891006 36.04% -========== NODE=pve01 ========== -SSH_STATUS=OK -HOST=pve01 -UTC_NOW=2026-07-22T14:53:53Z -=== FAILED_UNITS === -FAILED_UNIT_COUNT=3 -homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state -homelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01 -homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 === TARGET_UNITS === Result=exit-code ExecMainStatus=1 -ExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-backup-matrix-refresh.service LoadState=loaded ActiveState=failed SubState=failed -UnitFileState=static -ActiveEnterTimestamp= -InactiveEnterTimestamp=Wed 2026-07-22 10:11:15 MSK -Result=success -Id=homelab-backup-matrix-refresh.timer -LoadState=loaded -ActiveState=inactive -SubState=dead -UnitFileState=disabled -ActiveEnterTimestamp= -InactiveEnterTimestamp= + Result=exit-code ExecMainStatus=1 -ExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-vm160-remote-backup.service LoadState=loaded ActiveState=failed SubState=failed -UnitFileState=static -ActiveEnterTimestamp= -InactiveEnterTimestamp=Tue 2026-07-21 02:39:14 MSK -Result=success -Id=homelab-vm160-remote-backup.timer -LoadState=loaded -ActiveState=inactive -SubState=dead -UnitFileState=disabled -ActiveEnterTimestamp= -InactiveEnterTimestamp= + Result=exit-code ExecMainStatus=1 -ExecStart={ path=/usr/local/sbin/homelab-cluster-admin-pull ; argv[]=/usr/local/sbin/homelab-cluster-admin-pull ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-cluster-admin-pull.service LoadState=loaded ActiveState=failed SubState=failed -UnitFileState=static -ActiveEnterTimestamp= -InactiveEnterTimestamp=Tue 2026-07-21 02:37:04 MSK -Result=success -Id=homelab-cluster-admin-pull.timer -LoadState=loaded -ActiveState=inactive -SubState=dead -UnitFileState=disabled -ActiveEnterTimestamp= -InactiveEnterTimestamp= -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -Id=homelab-vm-local-dumps-2cloud.service -LoadState=loaded -ActiveState=inactive -SubState=dead -UnitFileState=static -ActiveEnterTimestamp= -InactiveEnterTimestamp=Sun 2026-07-19 10:13:40 MSK -Result=success -Id=homelab-vm-local-dumps-2cloud.timer -LoadState=loaded -ActiveState=active -SubState=waiting -UnitFileState=enabled -ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK -InactiveEnterTimestamp= -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-critical-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-critical-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -Id=homelab-mail-cloud-critical-upload.service -LoadState=loaded -ActiveState=inactive -SubState=dead -UnitFileState=static -ActiveEnterTimestamp= -InactiveEnterTimestamp=Wed 2026-07-22 06:46:47 MSK -Result=success -Id=homelab-mail-cloud-critical-upload.timer -LoadState=loaded -ActiveState=active -SubState=waiting -UnitFileState=enabled -ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK -InactiveEnterTimestamp= -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -Id=homelab-mail-cloud-edge-vm-upload.service -LoadState=loaded -ActiveState=inactive -SubState=dead -UnitFileState=static -ActiveEnterTimestamp= -InactiveEnterTimestamp=Sun 2026-07-19 09:41:56 MSK -Result=success -Id=homelab-mail-cloud-edge-vm-upload.timer -LoadState=loaded -ActiveState=active -SubState=waiting -UnitFileState=enabled -ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK -InactiveEnterTimestamp= -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -Id=homelab-mail-cloud-nextcloud-vm-upload.service -LoadState=loaded -ActiveState=inactive -SubState=dead -UnitFileState=static -ActiveEnterTimestamp= -InactiveEnterTimestamp=Wed 2026-07-22 07:45:01 MSK -Result=success -Id=homelab-mail-cloud-nextcloud-vm-upload.timer -LoadState=loaded -ActiveState=active -SubState=waiting -UnitFileState=enabled -ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK -InactiveEnterTimestamp= -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -Id=homelab-mail-cloud-restore-drill.service -LoadState=loaded -ActiveState=inactive -SubState=dead -UnitFileState=static -ActiveEnterTimestamp= -InactiveEnterTimestamp=Mon 2026-07-20 07:25:14 MSK -Result=success -Id=homelab-mail-cloud-restore-drill.timer -LoadState=loaded -ActiveState=active -SubState=waiting -UnitFileState=enabled -ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK -InactiveEnterTimestamp= -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-health-evidence ; argv[]=/usr/local/sbin/homelab-backup-health-evidence ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -Id=homelab-verified-backup-health.service -LoadState=loaded -ActiveState=inactive -SubState=dead -UnitFileState=static -ActiveEnterTimestamp= -InactiveEnterTimestamp=Wed 2026-07-22 06:40:29 MSK -Result=success -Id=homelab-verified-backup-health.timer -LoadState=loaded -ActiveState=active -SubState=waiting -UnitFileState=enabled -ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK -InactiveEnterTimestamp= -Result=success -ExecMainStatus=0 -ExecStart={ path=/usr/local/sbin/homelab-backup-sla-health ; argv[]=/usr/local/sbin/homelab-backup-sla-health ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } -Id=homelab-backup-sla-health.service -LoadState=loaded -ActiveState=inactive -SubState=dead -UnitFileState=static -ActiveEnterTimestamp= -InactiveEnterTimestamp=Wed 2026-07-22 08:10:42 MSK -Result=success -Id=homelab-backup-sla-health.timer -LoadState=loaded -ActiveState=active -SubState=waiting -UnitFileState=enabled -ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK -InactiveEnterTimestamp= -=== RELEVANT_TIMERS === -Thu 2026-07-23 06:38:07 MSK 12h Wed 2026-07-22 06:39:05 MSK 11h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service -Thu 2026-07-23 06:44:08 MSK 12h Wed 2026-07-22 06:44:25 MSK 11h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service -Thu 2026-07-23 07:58:35 MSK 14h Wed 2026-07-22 07:42:14 MSK 10h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service -Thu 2026-07-23 08:12:51 MSK 14h Wed 2026-07-22 08:10:41 MSK 9h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service -Sun 2026-07-26 09:46:53 MSK 3 days Sun 2026-07-19 09:32:29 MSK 3 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service -Sun 2026-07-26 10:23:39 MSK 3 days Sun 2026-07-19 10:06:07 MSK 3 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service -Mon 2026-07-27 07:34:27 MSK 4 days Mon 2026-07-20 07:25:14 MSK 2 days ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service -=== TARGET_SCRIPT_STATE === -FILE=/usr/local/sbin/homelab-backup-matrix-refresh TYPE=regular file MODE=755 OWNER=root:root SIZE=8714 MTIME=2026-07-10 13:02:04.846868245 +0300 -SCRIPT_SYNTAX=/usr/local/sbin/homelab-backup-matrix-refresh:FAIL -FILE=/usr/local/sbin/homelab-vm160-remote-backup TYPE=regular file MODE=755 OWNER=root:root SIZE=3140 MTIME=2026-07-18 01:38:18.170629793 +0300 -SCRIPT_SYNTAX=/usr/local/sbin/homelab-vm160-remote-backup:OK -FILE=/usr/local/sbin/homelab-cluster-admin-pull TYPE=regular file MODE=755 OWNER=root:root SIZE=2501 MTIME=2026-07-13 01:15:06.431144817 +0300 -SCRIPT_SYNTAX=/usr/local/sbin/homelab-cluster-admin-pull:OK -FILE=/usr/local/sbin/homelab-vm-local-dumps-2cloud TYPE=regular file MODE=700 OWNER=root:root SIZE=7272 MTIME=2026-07-06 18:49:49.740645050 +0300 -SCRIPT_SYNTAX=/usr/local/sbin/homelab-vm-local-dumps-2cloud:OK -FILE=/usr/local/sbin/homelab-mail-cloud-critical-upload TYPE=regular file MODE=700 OWNER=root:root SIZE=7836 MTIME=2026-07-17 14:03:50.373696143 +0300 -SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-critical-upload:OK -FILE=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=12899 MTIME=2026-07-13 21:27:22.164609073 +0300 -SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:OK -FILE=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=6542 MTIME=2026-06-30 15:00:33.588620116 +0300 -SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:OK -FILE=/usr/local/sbin/homelab-mail-cloud-restore-drill TYPE=regular file MODE=700 OWNER=root:root SIZE=7995 MTIME=2026-07-10 04:14:10.054957455 +0300 -SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-restore-drill:OK -=== CURRENT_BACKUP_HEALTH === -HEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt -HEALTH_FILE=/var/lib/homelab-health/backup-framework.txt -HEALTH_MTIME=2026-07-22 10:11:15.588809944 +0300 HEALTH_SIZE=803 +=== TARGET_FAILURES === +homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state +homelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01 +homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 +=== HEALTH_MARKERS === +FILE=/var/lib/homelab-health/backup-framework.txt MTIME=2026-07-22 10:11:15.588809944 +0300 SIZE=803 STATUS=OK -HEALTH_FILE=/var/lib/homelab-health/backup-sla.txt -HEALTH_MTIME=2026-07-22 17:45:04.576070554 +0300 HEALTH_SIZE=154 +TS=2026-07-22T07:11:15Z +TYPE=backup-matrix-generated +FILE=/var/lib/homelab-health/backup-sla.txt MTIME=2026-07-22 17:45:04.576070554 +0300 SIZE=154 STATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md -HEALTH_FILE=/var/lib/homelab-health/backup-coverage-matrix.txt -HEALTH_MTIME=2026-07-22 10:11:15.236509358 +0300 HEALTH_SIZE=808 +FILE=/var/lib/homelab-health/backup-coverage-matrix.txt MTIME=2026-07-22 10:11:15.236509358 +0300 SIZE=808 STATUS=OK -HEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt -HEALTH_MTIME=2026-07-22 10:11:15.694007276 +0300 HEALTH_SIZE=800 +TS=2026-07-22T07:11:15Z +TYPE=backup-matrix-generated +FILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt MTIME=2026-07-22 10:11:15.694007276 +0300 SIZE=800 STATUS=OK -HEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-retention.txt -HEALTH_MTIME=2026-07-22 10:11:15.711074323 +0300 HEALTH_SIZE=826 -STATUS=OK -HEALTH_FILE=/var/lib/homelab-health/vm-backup.txt -HEALTH_MTIME=2026-07-22 07:35:14.956502473 +0300 HEALTH_SIZE=170 -STATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active -HEALTH_FILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt -HEALTH_MTIME=2026-07-09 08:07:47.213452046 +0300 HEALTH_SIZE=217 +TS=2026-07-22T07:11:15Z +TYPE=backup-matrix-generated +FILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt MTIME=2026-07-09 08:07:47.213452046 +0300 SIZE=217 STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md -HEALTH_FILE=/var/lib/homelab-health/edge-vm-offhost-freshness.txt -HEALTH_MTIME=2026-07-22 04:46:23.189237991 +0300 HEALTH_SIZE=409 -STATUS=OK TS=2026-07-22T01:39:30Z TYPE=edge-vm-offhost-freshness BAD=0 VMID=130 SOURCE_NODE=pve03 DEST_NODE=pve02 SRC=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst DEST=/mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 SHA256=a8e5179614e02e6933617adb30cb95a35a1b5d17b30c9f00a2626cfa06b14710 COPY_STATUS=copied ZSTD_RC=0 VMA_RC=0 KEEP=4 -HEALTH_FILE=/var/lib/homelab-health/mail-cloud-restore-drill.txt -HEALTH_MTIME=2026-07-22 10:11:15.491775958 +0300 HEALTH_SIZE=830 -STATUS=STALE -=== RCLONE_REMOTE_NAMES === -pve01-mail-01: -pve01-mail-01-crypt: -pve01-mail-02: -pve01-mail-02-crypt: -pve01-mail-03: -pve01-mail-03-crypt: -pve01-mail-04: -pve01-mail-04-crypt: -pve01-mail-05: -pve01-mail-05-crypt: -pve02-mail-01: -pve02-mail-01-crypt: -pve02-mail-02: -pve02-mail-02-crypt: -pve02-mail-03: -pve02-mail-03-crypt: -pve02-mail-04: -pve02-mail-04-crypt: -pve03-mail-01: -pve03-mail-01-crypt: -pve03-mail-02: -pve03-mail-02-crypt: -pve03-mail-03: -pve03-mail-03-crypt: -pve03-mail-04: -pve03-mail-04-crypt: -=== ACTIVE_VZDUMP === -1599768 bash /usr/local/sbin/homelab-chat-run BACKUP-OFFHOST-CURRENT-STATE-212 read-only backup-audit -- /usr/bin/bash -c set -eu test "$(hostname -s)" = "pve01" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W="$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)" trap "rm -rf -- \"$W\"" EXIT BASE="https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history" curl -fsSL \ "$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json" \ -o "$W/204.json" jq -e " .command_id == \"HOMELAB-TAILS-FINAL-CLOSEOUT-204\" and .status == \"OK\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \"\") | contains(\"ALL_TAILS_CLOSED=YES\")) and ((.output // \"\") | contains(\"TAILS_FINAL_CLOSEOUT=COMPLETE\")) " "$W/204.json" >/dev/null curl -fsSL \ "$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json" \ -o "$W/122.json" jq -e " .command_id == \"HOMELAB-INDEPENDENT-FINAL-STATE-122\" and .status == \"OK\" and ((.rc | tonumber) == 0) and ((.output // \"\") | contains(\"HOMELAB_FINAL_STATE=BLOCKED\")) and ((.output // \"\") | contains(\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\")) " "$W/122.json" >/dev/null echo "PREVIOUS_RESULT_204=VERIFIED_OK" echo "HISTORICAL_BACKUP_BASELINE_122=VERIFIED" echo "REFERENCE_CHECK=OK" echo "ERROR_REGISTER_CHECK=OK" echo "AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "=== CURRENT_REFERENCE_STATE ===" STATE="/etc/homelab/reference/state/current.json" if test -s "$STATE" && jq -e "." "$STATE" >/dev/null; then jq -c "{status,next_action,updated_at}" "$STATE" else echo "REFERENCE_STATE=INVALID_OR_MISSING" fi echo "=== CLUSTER_NODES ===" pvesh get /nodes --output-format json | jq -c "sort_by(.node)[] | {node,status,uptime}" echo "=== PVE_BACKUP_JOBS ===" pvesh get /cluster/backup --output-format json | jq -c "sort_by(.id)" echo "=== STORAGE_STATUS ===" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo "========== NODE=$NODE ==========" if ssh "${SSH_OPTS[@]}" root@"$NODE" true then echo "SSH_STATUS=OK" ssh "${SSH_OPTS[@]}" root@"$NODE" " set -u echo HOST=\$(hostname -s) echo UTC_NOW=\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \ homelab-backup-matrix-refresh.service \ homelab-backup-matrix-refresh.timer \ homelab-vm160-remote-backup.service \ homelab-vm160-remote-backup.timer \ homelab-cluster-admin-pull.service \ homelab-cluster-admin-pull.timer \ homelab-vm-local-dumps-2cloud.service \ homelab-vm-local-dumps-2cloud.timer \ homelab-mail-cloud-critical-upload.service \ homelab-mail-cloud-critical-upload.timer \ homelab-mail-cloud-edge-vm-upload.service \ homelab-mail-cloud-edge-vm-upload.timer \ homelab-mail-cloud-nextcloud-vm-upload.service \ homelab-mail-cloud-nextcloud-vm-upload.timer \ homelab-mail-cloud-restore-drill.service \ homelab-mail-cloud-restore-drill.timer \ homelab-verified-backup-health.service \ homelab-verified-backup-health.timer \ homelab-backup-sla-health.service \ homelab-backup-sla-health.timer do if systemctl cat \"\$U\" >/dev/null 2>&1 then systemctl show \"\$U\" \ -p Id \ -p LoadState \ -p ActiveState \ -p SubState \ -p Result \ -p ExecMainStatus \ -p UnitFileState \ -p ActiveEnterTimestamp \ -p InactiveEnterTimestamp \ -p ExecStart \ --no-pager else echo UNIT_MISSING=\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in \ /usr/local/sbin/homelab-backup-matrix-refresh \ /usr/local/sbin/homelab-vm160-remote-backup \ /usr/local/sbin/homelab-cluster-admin-pull \ /usr/local/sbin/homelab-vm-local-dumps-2cloud \ /usr/local/sbin/homelab-mail-cloud-critical-upload \ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"\$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"\$F\" if /usr/bin/bash -n \"\$F\" 2>/dev/null then echo SCRIPT_SYNTAX=\$F:OK else echo SCRIPT_SYNTAX=\$F:FAIL fi else echo FILE_MISSING=\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \ /var/lib/homelab-health/unified-backup-matrix.txt \ /var/lib/homelab-health/backup-framework.txt \ /var/lib/homelab-health/backup-sla.txt \ /var/lib/homelab-health/backup-coverage-matrix.txt \ /var/lib/homelab-health/vm-local-dumps-cloud.txt \ /var/lib/homelab-health/vm-local-dumps-retention.txt \ /var/lib/homelab-health/vm-backup.txt \ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"\$H\" then echo HEALTH_FILE=\$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"\$H\" sed -n \"1p\" \"\$H\" else echo HEALTH_FILE_MISSING=\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf x \ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n\" \ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \ -u homelab-backup-matrix-refresh.service \ -u homelab-vm160-remote-backup.service \ -u homelab-cluster-admin-pull.service \ -n 120 \ --no-pager \ -o short-iso \ 2>/dev/null || true " else echo "SSH_STATUS=FAIL" fi done echo "CHANGES_MADE=NO" echo "SECRET_PRINTED=NO" echo "PRIVATE_ADDRESS_PRINTED=NO" echo "RESULT=OK" echo "BACKUP_OFFHOST_CURRENT_STATE=COMPLETE" -1599834 timeout --foreground --preserve-status 300 /usr/bin/bash -c set -eu test "$(hostname -s)" = "pve01" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W="$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)" trap "rm -rf -- \"$W\"" EXIT BASE="https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history" curl -fsSL \ "$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json" \ -o "$W/204.json" jq -e " .command_id == \"HOMELAB-TAILS-FINAL-CLOSEOUT-204\" and .status == \"OK\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \"\") | contains(\"ALL_TAILS_CLOSED=YES\")) and ((.output // \"\") | contains(\"TAILS_FINAL_CLOSEOUT=COMPLETE\")) " "$W/204.json" >/dev/null curl -fsSL \ "$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json" \ -o "$W/122.json" jq -e " .command_id == \"HOMELAB-INDEPENDENT-FINAL-STATE-122\" and .status == \"OK\" and ((.rc | tonumber) == 0) and ((.output // \"\") | contains(\"HOMELAB_FINAL_STATE=BLOCKED\")) and ((.output // \"\") | contains(\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\")) " "$W/122.json" >/dev/null echo "PREVIOUS_RESULT_204=VERIFIED_OK" echo "HISTORICAL_BACKUP_BASELINE_122=VERIFIED" echo "REFERENCE_CHECK=OK" echo "ERROR_REGISTER_CHECK=OK" echo "AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "=== CURRENT_REFERENCE_STATE ===" STATE="/etc/homelab/reference/state/current.json" if test -s "$STATE" && jq -e "." "$STATE" >/dev/null; then jq -c "{status,next_action,updated_at}" "$STATE" else echo "REFERENCE_STATE=INVALID_OR_MISSING" fi echo "=== CLUSTER_NODES ===" pvesh get /nodes --output-format json | jq -c "sort_by(.node)[] | {node,status,uptime}" echo "=== PVE_BACKUP_JOBS ===" pvesh get /cluster/backup --output-format json | jq -c "sort_by(.id)" echo "=== STORAGE_STATUS ===" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo "========== NODE=$NODE ==========" if ssh "${SSH_OPTS[@]}" root@"$NODE" true then echo "SSH_STATUS=OK" ssh "${SSH_OPTS[@]}" root@"$NODE" " set -u echo HOST=\$(hostname -s) echo UTC_NOW=\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \ homelab-backup-matrix-refresh.service \ homelab-backup-matrix-refresh.timer \ homelab-vm160-remote-backup.service \ homelab-vm160-remote-backup.timer \ homelab-cluster-admin-pull.service \ homelab-cluster-admin-pull.timer \ homelab-vm-local-dumps-2cloud.service \ homelab-vm-local-dumps-2cloud.timer \ homelab-mail-cloud-critical-upload.service \ homelab-mail-cloud-critical-upload.timer \ homelab-mail-cloud-edge-vm-upload.service \ homelab-mail-cloud-edge-vm-upload.timer \ homelab-mail-cloud-nextcloud-vm-upload.service \ homelab-mail-cloud-nextcloud-vm-upload.timer \ homelab-mail-cloud-restore-drill.service \ homelab-mail-cloud-restore-drill.timer \ homelab-verified-backup-health.service \ homelab-verified-backup-health.timer \ homelab-backup-sla-health.service \ homelab-backup-sla-health.timer do if systemctl cat \"\$U\" >/dev/null 2>&1 then systemctl show \"\$U\" \ -p Id \ -p LoadState \ -p ActiveState \ -p SubState \ -p Result \ -p ExecMainStatus \ -p UnitFileState \ -p ActiveEnterTimestamp \ -p InactiveEnterTimestamp \ -p ExecStart \ --no-pager else echo UNIT_MISSING=\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in \ /usr/local/sbin/homelab-backup-matrix-refresh \ /usr/local/sbin/homelab-vm160-remote-backup \ /usr/local/sbin/homelab-cluster-admin-pull \ /usr/local/sbin/homelab-vm-local-dumps-2cloud \ /usr/local/sbin/homelab-mail-cloud-critical-upload \ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"\$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"\$F\" if /usr/bin/bash -n \"\$F\" 2>/dev/null then echo SCRIPT_SYNTAX=\$F:OK else echo SCRIPT_SYNTAX=\$F:FAIL fi else echo FILE_MISSING=\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \ /var/lib/homelab-health/unified-backup-matrix.txt \ /var/lib/homelab-health/backup-framework.txt \ /var/lib/homelab-health/backup-sla.txt \ /var/lib/homelab-health/backup-coverage-matrix.txt \ /var/lib/homelab-health/vm-local-dumps-cloud.txt \ /var/lib/homelab-health/vm-local-dumps-retention.txt \ /var/lib/homelab-health/vm-backup.txt \ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"\$H\" then echo HEALTH_FILE=\$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"\$H\" sed -n \"1p\" \"\$H\" else echo HEALTH_FILE_MISSING=\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf x \ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n\" \ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \ -u homelab-backup-matrix-refresh.service \ -u homelab-vm160-remote-backup.service \ -u homelab-cluster-admin-pull.service \ -n 120 \ --no-pager \ -o short-iso \ 2>/dev/null || true " else echo "SSH_STATUS=FAIL" fi done echo "CHANGES_MADE=NO" echo "SECRET_PRINTED=NO" echo "PRIVATE_ADDRESS_PRINTED=NO" echo "RESULT=OK" echo "BACKUP_OFFHOST_CURRENT_STATE=COMPLETE" -1599836 /usr/bin/bash -c set -eu test "$(hostname -s)" = "pve01" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W="$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)" trap "rm -rf -- \"$W\"" EXIT BASE="https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history" curl -fsSL \ "$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json" \ -o "$W/204.json" jq -e " .command_id == \"HOMELAB-TAILS-FINAL-CLOSEOUT-204\" and .status == \"OK\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \"\") | contains(\"ALL_TAILS_CLOSED=YES\")) and ((.output // \"\") | contains(\"TAILS_FINAL_CLOSEOUT=COMPLETE\")) " "$W/204.json" >/dev/null curl -fsSL \ "$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json" \ -o "$W/122.json" jq -e " .command_id == \"HOMELAB-INDEPENDENT-FINAL-STATE-122\" and .status == \"OK\" and ((.rc | tonumber) == 0) and ((.output // \"\") | contains(\"HOMELAB_FINAL_STATE=BLOCKED\")) and ((.output // \"\") | contains(\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\")) " "$W/122.json" >/dev/null echo "PREVIOUS_RESULT_204=VERIFIED_OK" echo "HISTORICAL_BACKUP_BASELINE_122=VERIFIED" echo "REFERENCE_CHECK=OK" echo "ERROR_REGISTER_CHECK=OK" echo "AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "=== CURRENT_REFERENCE_STATE ===" STATE="/etc/homelab/reference/state/current.json" if test -s "$STATE" && jq -e "." "$STATE" >/dev/null; then jq -c "{status,next_action,updated_at}" "$STATE" else echo "REFERENCE_STATE=INVALID_OR_MISSING" fi echo "=== CLUSTER_NODES ===" pvesh get /nodes --output-format json | jq -c "sort_by(.node)[] | {node,status,uptime}" echo "=== PVE_BACKUP_JOBS ===" pvesh get /cluster/backup --output-format json | jq -c "sort_by(.id)" echo "=== STORAGE_STATUS ===" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo "========== NODE=$NODE ==========" if ssh "${SSH_OPTS[@]}" root@"$NODE" true then echo "SSH_STATUS=OK" ssh "${SSH_OPTS[@]}" root@"$NODE" " set -u echo HOST=\$(hostname -s) echo UTC_NOW=\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \ homelab-backup-matrix-refresh.service \ homelab-backup-matrix-refresh.timer \ homelab-vm160-remote-backup.service \ homelab-vm160-remote-backup.timer \ homelab-cluster-admin-pull.service \ homelab-cluster-admin-pull.timer \ homelab-vm-local-dumps-2cloud.service \ homelab-vm-local-dumps-2cloud.timer \ homelab-mail-cloud-critical-upload.service \ homelab-mail-cloud-critical-upload.timer \ homelab-mail-cloud-edge-vm-upload.service \ homelab-mail-cloud-edge-vm-upload.timer \ homelab-mail-cloud-nextcloud-vm-upload.service \ homelab-mail-cloud-nextcloud-vm-upload.timer \ homelab-mail-cloud-restore-drill.service \ homelab-mail-cloud-restore-drill.timer \ homelab-verified-backup-health.service \ homelab-verified-backup-health.timer \ homelab-backup-sla-health.service \ homelab-backup-sla-health.timer do if systemctl cat \"\$U\" >/dev/null 2>&1 then systemctl show \"\$U\" \ -p Id \ -p LoadState \ -p ActiveState \ -p SubState \ -p Result \ -p ExecMainStatus \ -p UnitFileState \ -p ActiveEnterTimestamp \ -p InactiveEnterTimestamp \ -p ExecStart \ --no-pager else echo UNIT_MISSING=\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in \ /usr/local/sbin/homelab-backup-matrix-refresh \ /usr/local/sbin/homelab-vm160-remote-backup \ /usr/local/sbin/homelab-cluster-admin-pull \ /usr/local/sbin/homelab-vm-local-dumps-2cloud \ /usr/local/sbin/homelab-mail-cloud-critical-upload \ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"\$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"\$F\" if /usr/bin/bash -n \"\$F\" 2>/dev/null then echo SCRIPT_SYNTAX=\$F:OK else echo SCRIPT_SYNTAX=\$F:FAIL fi else echo FILE_MISSING=\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \ /var/lib/homelab-health/unified-backup-matrix.txt \ /var/lib/homelab-health/backup-framework.txt \ /var/lib/homelab-health/backup-sla.txt \ /var/lib/homelab-health/backup-coverage-matrix.txt \ /var/lib/homelab-health/vm-local-dumps-cloud.txt \ /var/lib/homelab-health/vm-local-dumps-retention.txt \ /var/lib/homelab-health/vm-backup.txt \ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"\$H\" then echo HEALTH_FILE=\$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"\$H\" sed -n \"1p\" \"\$H\" else echo HEALTH_FILE_MISSING=\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf x \ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n\" \ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \ -u homelab-backup-matrix-refresh.service \ -u homelab-vm160-remote-backup.service \ -u homelab-cluster-admin-pull.service \ -n 120 \ --no-pager \ -o short-iso \ 2>/dev/null || true " else echo "SSH_STATUS=FAIL" fi done echo "CHANGES_MADE=NO" echo "SECRET_PRINTED=NO" echo "PRIVATE_ADDRESS_PRINTED=NO" echo "RESULT=OK" echo "BACKUP_OFFHOST_CURRENT_STATE=COMPLETE" -1603200 ssh -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes root@pve01 set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat "$U" >/dev/null 2>&1 then systemctl show "$U" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e "$F" then stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F" if /usr/bin/bash -n "$F" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f "$H" then echo HEALTH_FILE=$H stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H" sed -n "1p" "$H" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true -1603210 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat "$U" >/dev/null 2>&1 then systemctl show "$U" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e "$F" then stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F" if /usr/bin/bash -n "$F" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f "$H" then echo HEALTH_FILE=$H stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H" sed -n "1p" "$H" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true +FILE=/var/lib/homelab-health/vm-backup.txt MTIME=2026-07-22 07:35:14.956502473 +0300 SIZE=170 +STATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active === LOCAL_ARCHIVES === -ARCHIVE_COUNT=12 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120 -=== TARGET_UNIT_LOGS === -2026-07-22T05:48:02+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_STATUS=FAIL -2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_TOTAL=41 -2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_OK=30 -2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_BAD=6 -2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_UNMAPPED=0 -2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_DISABLED=5 -2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -2026-07-22T05:48:03+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -2026-07-22T06:11:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_STATUS=FAIL -2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_TOTAL=41 -2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_OK=30 -2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_BAD=6 -2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_UNMAPPED=0 -2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_DISABLED=5 -2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -2026-07-22T06:11:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -2026-07-22T06:40:29+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_STATUS=FAIL -2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_TOTAL=41 -2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_OK=30 -2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_BAD=6 -2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_UNMAPPED=0 -2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_DISABLED=5 -2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -2026-07-22T06:40:30+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -2026-07-22T07:02:40+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_STATUS=FAIL -2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_TOTAL=41 -2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_OK=30 -2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_BAD=6 -2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_UNMAPPED=0 -2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_DISABLED=5 -2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -2026-07-22T07:02:41+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -2026-07-22T07:35:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_STATUS=FAIL -2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_TOTAL=41 -2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_OK=30 -2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_BAD=6 -2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_UNMAPPED=0 -2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_DISABLED=5 -2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -2026-07-22T07:35:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -2026-07-22T08:10:42+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_STATUS=FAIL -2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_TOTAL=41 -2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_OK=30 -2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_BAD=6 -2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_UNMAPPED=0 -2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_DISABLED=5 -2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -2026-07-22T08:10:43+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -2026-07-22T08:37:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_STATUS=FAIL -2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_TOTAL=41 -2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_OK=30 -2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_BAD=6 -2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_UNMAPPED=0 -2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_DISABLED=5 -2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -2026-07-22T08:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -2026-07-22T08:40:15+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_STATUS=FAIL -2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_TOTAL=41 -2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_OK=30 -2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_BAD=6 -2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_UNMAPPED=0 -2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_DISABLED=5 -2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -2026-07-22T08:40:16+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -2026-07-22T08:53:36+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_STATUS=FAIL -2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_TOTAL=41 -2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_OK=30 -2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_BAD=6 -2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_UNMAPPED=0 -2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_DISABLED=5 -2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -2026-07-22T08:53:37+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... -2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_STATUS=FAIL -2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41 -2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30 -2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6 -2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0 -2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5 -2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json -2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered -2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE -2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. -2026-07-22T10:11:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. -========== NODE=pve02 ========== -SSH_STATUS=OK -HOST=pve02 -UTC_NOW=2026-07-22T14:53:54Z -=== FAILED_UNITS === -FAILED_UNIT_COUNT=0 -=== TARGET_UNITS === -UNIT_MISSING=homelab-backup-matrix-refresh.service -UNIT_MISSING=homelab-backup-matrix-refresh.timer -UNIT_MISSING=homelab-vm160-remote-backup.service -UNIT_MISSING=homelab-vm160-remote-backup.timer -UNIT_MISSING=homelab-cluster-admin-pull.service -UNIT_MISSING=homelab-cluster-admin-pull.timer -UNIT_MISSING=homelab-vm-local-dumps-2cloud.service -UNIT_MISSING=homelab-vm-local-dumps-2cloud.timer -UNIT_MISSING=homelab-mail-cloud-critical-upload.service -UNIT_MISSING=homelab-mail-cloud-critical-upload.timer -UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service -UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer -UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service -UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer -UNIT_MISSING=homelab-mail-cloud-restore-drill.service -UNIT_MISSING=homelab-mail-cloud-restore-drill.timer -UNIT_MISSING=homelab-verified-backup-health.service -UNIT_MISSING=homelab-verified-backup-health.timer -UNIT_MISSING=homelab-backup-sla-health.service -UNIT_MISSING=homelab-backup-sla-health.timer -=== RELEVANT_TIMERS === -=== TARGET_SCRIPT_STATE === -FILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh -FILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup -FILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull -FILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud -FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload -FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload -FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload -FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill -=== CURRENT_BACKUP_HEALTH === -HEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt -=== RCLONE_REMOTE_NAMES === -pve01-mail-01: -pve01-mail-01-crypt: -pve01-mail-02: -pve01-mail-02-crypt: -pve01-mail-03: -pve01-mail-03-crypt: -pve01-mail-04: -pve01-mail-04-crypt: -pve01-mail-05: -pve01-mail-05-crypt: -pve02-mail-01: -pve02-mail-01-crypt: -pve02-mail-02: -pve02-mail-02-crypt: -pve02-mail-03: -pve02-mail-03-crypt: -pve02-mail-04: -pve02-mail-04-crypt: -pve03-mail-01: -pve03-mail-01-crypt: -pve03-mail-02: -pve03-mail-02-crypt: -pve03-mail-03: -pve03-mail-03-crypt: -pve03-mail-04: -pve03-mail-04-crypt: -=== ACTIVE_VZDUMP === -3930634 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat "$U" >/dev/null 2>&1 then systemctl show "$U" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e "$F" then stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F" if /usr/bin/bash -n "$F" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f "$H" then echo HEALTH_FILE=$H stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H" sed -n "1p" "$H" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true -=== LOCAL_ARCHIVES === -ARCHIVE_COUNT=10 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst SIZE=284443608 MTIME=2026-07-19T03:47:41.8525503200 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst SIZE=286538526 MTIME=2026-07-20T03:30:18.4386603270 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst SIZE=288349157 MTIME=2026-07-21T03:30:21.2480085730 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_22-03_30_01.tar.zst SIZE=290059742 MTIME=2026-07-22T03:30:22.8748248510 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst SIZE=220049041 MTIME=2026-07-19T03:47:58.9256407960 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst SIZE=220065825 MTIME=2026-07-20T03:30:30.8287264090 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst SIZE=220080970 MTIME=2026-07-21T03:30:33.3910787180 -ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_22-03_30_23.tar.zst SIZE=220095608 MTIME=2026-07-22T03:30:40.6809207660 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst SIZE=868048369 MTIME=2026-06-30T03:31:08.4903256780 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst SIZE=997503422 MTIME=2026-07-01T03:31:11.1700059310 -=== TARGET_UNIT_LOGS === --- No entries -- -========== NODE=pve03 ========== -SSH_STATUS=OK -HOST=pve03 -UTC_NOW=2026-07-22T14:53:55Z -=== FAILED_UNITS === -FAILED_UNIT_COUNT=1 -homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image -=== TARGET_UNITS === -UNIT_MISSING=homelab-backup-matrix-refresh.service -UNIT_MISSING=homelab-backup-matrix-refresh.timer -UNIT_MISSING=homelab-vm160-remote-backup.service -UNIT_MISSING=homelab-vm160-remote-backup.timer -UNIT_MISSING=homelab-cluster-admin-pull.service -UNIT_MISSING=homelab-cluster-admin-pull.timer -UNIT_MISSING=homelab-vm-local-dumps-2cloud.service -UNIT_MISSING=homelab-vm-local-dumps-2cloud.timer -UNIT_MISSING=homelab-mail-cloud-critical-upload.service -UNIT_MISSING=homelab-mail-cloud-critical-upload.timer -UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service -UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer -UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service -UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer -UNIT_MISSING=homelab-mail-cloud-restore-drill.service -UNIT_MISSING=homelab-mail-cloud-restore-drill.timer -UNIT_MISSING=homelab-verified-backup-health.service -UNIT_MISSING=homelab-verified-backup-health.timer -UNIT_MISSING=homelab-backup-sla-health.service -UNIT_MISSING=homelab-backup-sla-health.timer -=== RELEVANT_TIMERS === -=== TARGET_SCRIPT_STATE === -FILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh -FILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup -FILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull -FILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud -FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload -FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload -FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload -FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill -=== CURRENT_BACKUP_HEALTH === -HEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt -HEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt -=== RCLONE_REMOTE_NAMES === -pve01-mail-01: -pve01-mail-01-crypt: -pve01-mail-02: -pve01-mail-02-crypt: -pve01-mail-03: -pve01-mail-03-crypt: -pve01-mail-04: -pve01-mail-04-crypt: -pve01-mail-05: -pve01-mail-05-crypt: -pve02-mail-01: -pve02-mail-01-crypt: -pve02-mail-02: -pve02-mail-02-crypt: -pve02-mail-03: -pve02-mail-03-crypt: -pve02-mail-04: -pve02-mail-04-crypt: -pve03-mail-01: -pve03-mail-01-crypt: -pve03-mail-02: -pve03-mail-02-crypt: -pve03-mail-03: -pve03-mail-03-crypt: -pve03-mail-04: -pve03-mail-04-crypt: -=== ACTIVE_VZDUMP === -2313989 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat "$U" >/dev/null 2>&1 then systemctl show "$U" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e "$F" then stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F" if /usr/bin/bash -n "$F" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f "$H" then echo HEALTH_FILE=$H stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H" sed -n "1p" "$H" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true -=== LOCAL_ARCHIVES === -ARCHIVE_COUNT=8 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst SIZE=11013723404 MTIME=2026-07-19T03:33:53.3463878040 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst SIZE=11116472459 MTIME=2026-07-20T03:33:53.2839064370 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst SIZE=11043834894 MTIME=2026-07-21T03:33:54.3725716230 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 MTIME=2026-07-22T03:33:58.8961071480 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst SIZE=838951976 MTIME=2026-07-19T03:34:33.1034859540 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst SIZE=838817925 MTIME=2026-07-20T03:34:37.1719936670 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst SIZE=1137072350 MTIME=2026-07-21T03:34:37.6276811360 -ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_22-03_33_59.vma.zst SIZE=1177407443 MTIME=2026-07-22T03:34:44.3412293530 -=== TARGET_UNIT_LOGS === --- No entries -- +ARCHIVE=vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400 +ARCHIVE=vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720 +ARCHIVE=vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330 +ARCHIVE=vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500 +ARCHIVE=vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730 +ARCHIVE=vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440 +ARCHIVE=vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290 +ARCHIVE=vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120 +ARCHIVE=vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020 +ARCHIVE=vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470 +ARCHIVE=vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950 +ARCHIVE=vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120 +=== RECENT_ERRORS === +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. +MATRIX_STATE_STATUS=FAIL +homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE +homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. +Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. CHANGES_MADE=NO -SECRET_PRINTED=NO -PRIVATE_ADDRESS_PRINTED=NO +REFERENCE_CHECK=OK +ERROR_REGISTER_CHECK=OK RESULT=OK -BACKUP_OFFHOST_CURRENT_STATE=COMPLETE +BACKUP_OFFHOST_DELTA=COMPLETE OUTPUT_END CHAT_OUTPUT_END