homelab-admin/homelab-public-context
1
0
Fork 0
Code Activity
Files
assistant-runtime
homelab-public-context/runtime/history/BACKUP-OFFHOST-CURRENT-STATE-212.json
T
homelab-runtime-publisher 09825fe96e runtime: publish BACKUP-OFFHOST-CURRENT-STATE-212
2026-07-22 14:53:56 +00:00

31 lines
89 KiB
JSON
Raw Permalink Blame History

{
"schema_version": 1,
"channel": "homelab-runtime",
"command_id": "BACKUP-OFFHOST-CURRENT-STATE-212",
"status": "OK",
"rc": 0,
"host": "pve01",
"mode": "read-only",
"component": "backup-audit",
"started_at_utc": "2026-07-22T14:53:51Z",
"finished_at_utc": "2026-07-22T14:53:55Z",
"reference_register_checked": true,
"reference_sha256": "5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e",
"error_register_checked": true,
"error_register_sha256": "08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752",
"command_sha256": "5b9f985ab6daceaad18e56d2da3dde0d2dba7eefdfe7f75e83eee6bef0a0ad8a",
"duplicate_failed_command_blocked": false,
"block_reason": null,
"execution_started": true,
"changes_made": false,
"sanitized": true,
"secrets_included": false,
"private_addresses_included": false,
"raw_evidence_retained_locally": true,
"raw_evidence_sha256": "9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2",
"sanitized_output_sha256": "9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2",
"output_truncated_in_json": false,
"full_sanitized_output_url": "https://git.gram1.ru/.well-known/homelab-runtime/latest.txt",
"output": "validation=PASS\nPREVIOUS_RESULT_204=VERIFIED_OK\nHISTORICAL_BACKUP_BASELINE_122=VERIFIED\nREFERENCE_CHECK=OK\nERROR_REGISTER_CHECK=OK\nAUDIT_GENERATED_AT=2026-07-22T14:53:51Z\n=== CURRENT_REFERENCE_STATE ===\n{\"status\":\"AUTO_REFRESH_READY\",\"next_action\":null,\"updated_at\":\"2026-07-22T14:22:14+00:00\"}\n=== CLUSTER_NODES ===\n{\"node\":\"pve01\",\"status\":\"online\",\"uptime\":540902}\n{\"node\":\"pve02\",\"status\":\"online\",\"uptime\":540921}\n{\"node\":\"pve03\",\"status\":\"online\",\"uptime\":540774}\n=== PVE_BACKUP_JOBS ===\n[{\"compress\":\"zstd\",\"enabled\":1,\"id\":\"homelab-nightly-all\",\"mailnotification\":\"failure\",\"mode\":\"snapshot\",\"next-run\":1784766600,\"prune-backups\":{\"keep-daily\":\"4\"},\"schedule\":\"03:30\",\"storage\":\"local\",\"type\":\"vzdump\",\"vmid\":\"110,111,112,113,130,150,190\"}]\n=== STORAGE_STATUS ===\nName Type Status Total (KiB) Used (KiB) Available (KiB) %\nlocal dir active 102626232 42074800 55292168 41.00%\nlocal-lvm lvmthin active 848797696 305906689 542891006 36.04%\n========== NODE=pve01 ==========\nSSH_STATUS=OK\nHOST=pve01\nUTC_NOW=2026-07-22T14:53:53Z\n=== FAILED_UNITS ===\nFAILED_UNIT_COUNT=3\nhomelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state\nhomelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01\nhomelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03\n=== TARGET_UNITS ===\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-backup-matrix-refresh.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 10:11:15 MSK\nResult=success\nId=homelab-backup-matrix-refresh.timer\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=disabled\nActiveEnterTimestamp=\nInactiveEnterTimestamp=\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-vm160-remote-backup.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Tue 2026-07-21 02:39:14 MSK\nResult=success\nId=homelab-vm160-remote-backup.timer\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=disabled\nActiveEnterTimestamp=\nInactiveEnterTimestamp=\nResult=exit-code\nExecMainStatus=1\nExecStart={ path=/usr/local/sbin/homelab-cluster-admin-pull ; argv[]=/usr/local/sbin/homelab-cluster-admin-pull ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-cluster-admin-pull.service\nLoadState=loaded\nActiveState=failed\nSubState=failed\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Tue 2026-07-21 02:37:04 MSK\nResult=success\nId=homelab-cluster-admin-pull.timer\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=disabled\nActiveEnterTimestamp=\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-vm-local-dumps-2cloud.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Sun 2026-07-19 10:13:40 MSK\nResult=success\nId=homelab-vm-local-dumps-2cloud.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-critical-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-critical-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-critical-upload.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 06:46:47 MSK\nResult=success\nId=homelab-mail-cloud-critical-upload.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-edge-vm-upload.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Sun 2026-07-19 09:41:56 MSK\nResult=success\nId=homelab-mail-cloud-edge-vm-upload.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-nextcloud-vm-upload.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 07:45:01 MSK\nResult=success\nId=homelab-mail-cloud-nextcloud-vm-upload.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-mail-cloud-restore-drill.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Mon 2026-07-20 07:25:14 MSK\nResult=success\nId=homelab-mail-cloud-restore-drill.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-health-evidence ; argv[]=/usr/local/sbin/homelab-backup-health-evidence ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-verified-backup-health.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 06:40:29 MSK\nResult=success\nId=homelab-verified-backup-health.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\nResult=success\nExecMainStatus=0\nExecStart={ path=/usr/local/sbin/homelab-backup-sla-health ; argv[]=/usr/local/sbin/homelab-backup-sla-health ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }\nId=homelab-backup-sla-health.service\nLoadState=loaded\nActiveState=inactive\nSubState=dead\nUnitFileState=static\nActiveEnterTimestamp=\nInactiveEnterTimestamp=Wed 2026-07-22 08:10:42 MSK\nResult=success\nId=homelab-backup-sla-health.timer\nLoadState=loaded\nActiveState=active\nSubState=waiting\nUnitFileState=enabled\nActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK\nInactiveEnterTimestamp=\n=== RELEVANT_TIMERS ===\nThu 2026-07-23 06:38:07 MSK 12h Wed 2026-07-22 06:39:05 MSK 11h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service\nThu 2026-07-23 06:44:08 MSK 12h Wed 2026-07-22 06:44:25 MSK 11h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service\nThu 2026-07-23 07:58:35 MSK 14h Wed 2026-07-22 07:42:14 MSK 10h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service\nThu 2026-07-23 08:12:51 MSK 14h Wed 2026-07-22 08:10:41 MSK 9h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service\nSun 2026-07-26 09:46:53 MSK 3 days Sun 2026-07-19 09:32:29 MSK 3 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service\nSun 2026-07-26 10:23:39 MSK 3 days Sun 2026-07-19 10:06:07 MSK 3 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service\nMon 2026-07-27 07:34:27 MSK 4 days Mon 2026-07-20 07:25:14 MSK 2 days ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service\n=== TARGET_SCRIPT_STATE ===\nFILE=/usr/local/sbin/homelab-backup-matrix-refresh TYPE=regular file MODE=755 OWNER=root:root SIZE=8714 MTIME=2026-07-10 13:02:04.846868245 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-backup-matrix-refresh:FAIL\nFILE=/usr/local/sbin/homelab-vm160-remote-backup TYPE=regular file MODE=755 OWNER=root:root SIZE=3140 MTIME=2026-07-18 01:38:18.170629793 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-vm160-remote-backup:OK\nFILE=/usr/local/sbin/homelab-cluster-admin-pull TYPE=regular file MODE=755 OWNER=root:root SIZE=2501 MTIME=2026-07-13 01:15:06.431144817 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-cluster-admin-pull:OK\nFILE=/usr/local/sbin/homelab-vm-local-dumps-2cloud TYPE=regular file MODE=700 OWNER=root:root SIZE=7272 MTIME=2026-07-06 18:49:49.740645050 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-vm-local-dumps-2cloud:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-critical-upload TYPE=regular file MODE=700 OWNER=root:root SIZE=7836 MTIME=2026-07-17 14:03:50.373696143 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-critical-upload:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=12899 MTIME=2026-07-13 21:27:22.164609073 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=6542 MTIME=2026-06-30 15:00:33.588620116 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:OK\nFILE=/usr/local/sbin/homelab-mail-cloud-restore-drill TYPE=regular file MODE=700 OWNER=root:root SIZE=7995 MTIME=2026-07-10 04:14:10.054957455 +0300\nSCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-restore-drill:OK\n=== CURRENT_BACKUP_HEALTH ===\nHEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt\nHEALTH_FILE=/var/lib/homelab-health/backup-framework.txt\nHEALTH_MTIME=2026-07-22 10:11:15.588809944 +0300 HEALTH_SIZE=803\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/backup-sla.txt\nHEALTH_MTIME=2026-07-22 17:45:04.576070554 +0300 HEALTH_SIZE=154\nSTATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md\nHEALTH_FILE=/var/lib/homelab-health/backup-coverage-matrix.txt\nHEALTH_MTIME=2026-07-22 10:11:15.236509358 +0300 HEALTH_SIZE=808\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt\nHEALTH_MTIME=2026-07-22 10:11:15.694007276 +0300 HEALTH_SIZE=800\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-retention.txt\nHEALTH_MTIME=2026-07-22 10:11:15.711074323 +0300 HEALTH_SIZE=826\nSTATUS=OK\nHEALTH_FILE=/var/lib/homelab-health/vm-backup.txt\nHEALTH_MTIME=2026-07-22 07:35:14.956502473 +0300 HEALTH_SIZE=170\nSTATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active\nHEALTH_FILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\nHEALTH_MTIME=2026-07-09 08:07:47.213452046 +0300 HEALTH_SIZE=217\nSTATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md\nHEALTH_FILE=/var/lib/homelab-health/edge-vm-offhost-freshness.txt\nHEALTH_MTIME=2026-07-22 04:46:23.189237991 +0300 HEALTH_SIZE=409\nSTATUS=OK TS=2026-07-22T01:39:30Z TYPE=edge-vm-offhost-freshness BAD=0 VMID=130 SOURCE_NODE=pve03 DEST_NODE=pve02 SRC=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst DEST=/mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 SHA256=a8e5179614e02e6933617adb30cb95a35a1b5d17b30c9f00a2626cfa06b14710 COPY_STATUS=copied ZSTD_RC=0 VMA_RC=0 KEEP=4\nHEALTH_FILE=/var/lib/homelab-health/mail-cloud-restore-drill.txt\nHEALTH_MTIME=2026-07-22 10:11:15.491775958 +0300 HEALTH_SIZE=830\nSTATUS=STALE\n=== RCLONE_REMOTE_NAMES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== ACTIVE_VZDUMP ===\n1599768 bash /usr/local/sbin/homelab-chat-run BACKUP-OFFHOST-CURRENT-STATE-212 read-only backup-audit -- /usr/bin/bash -c set -eu test \"$(hostname -s)\" = \"pve01\" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W=\"$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)\" trap \"rm -rf -- \\\"$W\\\"\" EXIT BASE=\"https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history\" curl -fsSL \\ \"$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json\" \\ -o \"$W/204.json\" jq -e \" .command_id == \\\"HOMELAB-TAILS-FINAL-CLOSEOUT-204\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \\\"\\\") | contains(\\\"ALL_TAILS_CLOSED=YES\\\")) and ((.output // \\\"\\\") | contains(\\\"TAILS_FINAL_CLOSEOUT=COMPLETE\\\")) \" \"$W/204.json\" >/dev/null curl -fsSL \\ \"$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json\" \\ -o \"$W/122.json\" jq -e \" .command_id == \\\"HOMELAB-INDEPENDENT-FINAL-STATE-122\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and ((.output // \\\"\\\") | contains(\\\"HOMELAB_FINAL_STATE=BLOCKED\\\")) and ((.output // \\\"\\\") | contains(\\\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\\\")) \" \"$W/122.json\" >/dev/null echo \"PREVIOUS_RESULT_204=VERIFIED_OK\" echo \"HISTORICAL_BACKUP_BASELINE_122=VERIFIED\" echo \"REFERENCE_CHECK=OK\" echo \"ERROR_REGISTER_CHECK=OK\" echo \"AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" echo \"=== CURRENT_REFERENCE_STATE ===\" STATE=\"/etc/homelab/reference/state/current.json\" if test -s \"$STATE\" && jq -e \".\" \"$STATE\" >/dev/null; then jq -c \"{status,next_action,updated_at}\" \"$STATE\" else echo \"REFERENCE_STATE=INVALID_OR_MISSING\" fi echo \"=== CLUSTER_NODES ===\" pvesh get /nodes --output-format json | jq -c \"sort_by(.node)[] | {node,status,uptime}\" echo \"=== PVE_BACKUP_JOBS ===\" pvesh get /cluster/backup --output-format json | jq -c \"sort_by(.id)\" echo \"=== STORAGE_STATUS ===\" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo \"========== NODE=$NODE ==========\" if ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" true then echo \"SSH_STATUS=OK\" ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" \" set -u echo HOST=\\$(hostname -s) echo UTC_NOW=\\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \\ homelab-backup-matrix-refresh.service \\ homelab-backup-matrix-refresh.timer \\ homelab-vm160-remote-backup.service \\ homelab-vm160-remote-backup.timer \\ homelab-cluster-admin-pull.service \\ homelab-cluster-admin-pull.timer \\ homelab-vm-local-dumps-2cloud.service \\ homelab-vm-local-dumps-2cloud.timer \\ homelab-mail-cloud-critical-upload.service \\ homelab-mail-cloud-critical-upload.timer \\ homelab-mail-cloud-edge-vm-upload.service \\ homelab-mail-cloud-edge-vm-upload.timer \\ homelab-mail-cloud-nextcloud-vm-upload.service \\ homelab-mail-cloud-nextcloud-vm-upload.timer \\ homelab-mail-cloud-restore-drill.service \\ homelab-mail-cloud-restore-drill.timer \\ homelab-verified-backup-health.service \\ homelab-verified-backup-health.timer \\ homelab-backup-sla-health.service \\ homelab-backup-sla-health.timer do if systemctl cat \\\"\\$U\\\" >/dev/null 2>&1 then systemctl show \\\"\\$U\\\" \\ -p Id \\ -p LoadState \\ -p ActiveState \\ -p SubState \\ -p Result \\ -p ExecMainStatus \\ -p UnitFileState \\ -p ActiveEnterTimestamp \\ -p InactiveEnterTimestamp \\ -p ExecStart \\ --no-pager else echo UNIT_MISSING=\\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \\\"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\\\" || true echo === TARGET_SCRIPT_STATE === for F in \\ /usr/local/sbin/homelab-backup-matrix-refresh \\ /usr/local/sbin/homelab-vm160-remote-backup \\ /usr/local/sbin/homelab-cluster-admin-pull \\ /usr/local/sbin/homelab-vm-local-dumps-2cloud \\ /usr/local/sbin/homelab-mail-cloud-critical-upload \\ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \\\"\\$F\\\" then stat -c \\\"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\\\" \\\"\\$F\\\" if /usr/bin/bash -n \\\"\\$F\\\" 2>/dev/null then echo SCRIPT_SYNTAX=\\$F:OK else echo SCRIPT_SYNTAX=\\$F:FAIL fi else echo FILE_MISSING=\\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \\ /var/lib/homelab-health/unified-backup-matrix.txt \\ /var/lib/homelab-health/backup-framework.txt \\ /var/lib/homelab-health/backup-sla.txt \\ /var/lib/homelab-health/backup-coverage-matrix.txt \\ /var/lib/homelab-health/vm-local-dumps-cloud.txt \\ /var/lib/homelab-health/vm-local-dumps-retention.txt \\ /var/lib/homelab-health/vm-backup.txt \\ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \\ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \\ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \\\"\\$H\\\" then echo HEALTH_FILE=\\$H stat -c \\\"HEALTH_MTIME=%y HEALTH_SIZE=%s\\\" \\\"\\$H\\\" sed -n \\\"1p\\\" \\\"\\$H\\\" else echo HEALTH_FILE_MISSING=\\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \\\"[v]zdump\\\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf x \\ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf \\\"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\\\" \\ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \\ -u homelab-backup-matrix-refresh.service \\ -u homelab-vm160-remote-backup.service \\ -u homelab-cluster-admin-pull.service \\ -n 120 \\ --no-pager \\ -o short-iso \\ 2>/dev/null || true \" else echo \"SSH_STATUS=FAIL\" fi done echo \"CHANGES_MADE=NO\" echo \"SECRET_PRINTED=NO\" echo \"PRIVATE_ADDRESS_PRINTED=NO\" echo \"RESULT=OK\" echo \"BACKUP_OFFHOST_CURRENT_STATE=COMPLETE\" \n1599834 timeout --foreground --preserve-status 300 /usr/bin/bash -c set -eu test \"$(hostname -s)\" = \"pve01\" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W=\"$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)\" trap \"rm -rf -- \\\"$W\\\"\" EXIT BASE=\"https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history\" curl -fsSL \\ \"$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json\" \\ -o \"$W/204.json\" jq -e \" .command_id == \\\"HOMELAB-TAILS-FINAL-CLOSEOUT-204\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \\\"\\\") | contains(\\\"ALL_TAILS_CLOSED=YES\\\")) and ((.output // \\\"\\\") | contains(\\\"TAILS_FINAL_CLOSEOUT=COMPLETE\\\")) \" \"$W/204.json\" >/dev/null curl -fsSL \\ \"$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json\" \\ -o \"$W/122.json\" jq -e \" .command_id == \\\"HOMELAB-INDEPENDENT-FINAL-STATE-122\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and ((.output // \\\"\\\") | contains(\\\"HOMELAB_FINAL_STATE=BLOCKED\\\")) and ((.output // \\\"\\\") | contains(\\\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\\\")) \" \"$W/122.json\" >/dev/null echo \"PREVIOUS_RESULT_204=VERIFIED_OK\" echo \"HISTORICAL_BACKUP_BASELINE_122=VERIFIED\" echo \"REFERENCE_CHECK=OK\" echo \"ERROR_REGISTER_CHECK=OK\" echo \"AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" echo \"=== CURRENT_REFERENCE_STATE ===\" STATE=\"/etc/homelab/reference/state/current.json\" if test -s \"$STATE\" && jq -e \".\" \"$STATE\" >/dev/null; then jq -c \"{status,next_action,updated_at}\" \"$STATE\" else echo \"REFERENCE_STATE=INVALID_OR_MISSING\" fi echo \"=== CLUSTER_NODES ===\" pvesh get /nodes --output-format json | jq -c \"sort_by(.node)[] | {node,status,uptime}\" echo \"=== PVE_BACKUP_JOBS ===\" pvesh get /cluster/backup --output-format json | jq -c \"sort_by(.id)\" echo \"=== STORAGE_STATUS ===\" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo \"========== NODE=$NODE ==========\" if ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" true then echo \"SSH_STATUS=OK\" ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" \" set -u echo HOST=\\$(hostname -s) echo UTC_NOW=\\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \\ homelab-backup-matrix-refresh.service \\ homelab-backup-matrix-refresh.timer \\ homelab-vm160-remote-backup.service \\ homelab-vm160-remote-backup.timer \\ homelab-cluster-admin-pull.service \\ homelab-cluster-admin-pull.timer \\ homelab-vm-local-dumps-2cloud.service \\ homelab-vm-local-dumps-2cloud.timer \\ homelab-mail-cloud-critical-upload.service \\ homelab-mail-cloud-critical-upload.timer \\ homelab-mail-cloud-edge-vm-upload.service \\ homelab-mail-cloud-edge-vm-upload.timer \\ homelab-mail-cloud-nextcloud-vm-upload.service \\ homelab-mail-cloud-nextcloud-vm-upload.timer \\ homelab-mail-cloud-restore-drill.service \\ homelab-mail-cloud-restore-drill.timer \\ homelab-verified-backup-health.service \\ homelab-verified-backup-health.timer \\ homelab-backup-sla-health.service \\ homelab-backup-sla-health.timer do if systemctl cat \\\"\\$U\\\" >/dev/null 2>&1 then systemctl show \\\"\\$U\\\" \\ -p Id \\ -p LoadState \\ -p ActiveState \\ -p SubState \\ -p Result \\ -p ExecMainStatus \\ -p UnitFileState \\ -p ActiveEnterTimestamp \\ -p InactiveEnterTimestamp \\ -p ExecStart \\ --no-pager else echo UNIT_MISSING=\\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \\\"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\\\" || true echo === TARGET_SCRIPT_STATE === for F in \\ /usr/local/sbin/homelab-backup-matrix-refresh \\ /usr/local/sbin/homelab-vm160-remote-backup \\ /usr/local/sbin/homelab-cluster-admin-pull \\ /usr/local/sbin/homelab-vm-local-dumps-2cloud \\ /usr/local/sbin/homelab-mail-cloud-critical-upload \\ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \\\"\\$F\\\" then stat -c \\\"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\\\" \\\"\\$F\\\" if /usr/bin/bash -n \\\"\\$F\\\" 2>/dev/null then echo SCRIPT_SYNTAX=\\$F:OK else echo SCRIPT_SYNTAX=\\$F:FAIL fi else echo FILE_MISSING=\\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \\ /var/lib/homelab-health/unified-backup-matrix.txt \\ /var/lib/homelab-health/backup-framework.txt \\ /var/lib/homelab-health/backup-sla.txt \\ /var/lib/homelab-health/backup-coverage-matrix.txt \\ /var/lib/homelab-health/vm-local-dumps-cloud.txt \\ /var/lib/homelab-health/vm-local-dumps-retention.txt \\ /var/lib/homelab-health/vm-backup.txt \\ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \\ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \\ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \\\"\\$H\\\" then echo HEALTH_FILE=\\$H stat -c \\\"HEALTH_MTIME=%y HEALTH_SIZE=%s\\\" \\\"\\$H\\\" sed -n \\\"1p\\\" \\\"\\$H\\\" else echo HEALTH_FILE_MISSING=\\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \\\"[v]zdump\\\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf x \\ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf \\\"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\\\" \\ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \\ -u homelab-backup-matrix-refresh.service \\ -u homelab-vm160-remote-backup.service \\ -u homelab-cluster-admin-pull.service \\ -n 120 \\ --no-pager \\ -o short-iso \\ 2>/dev/null || true \" else echo \"SSH_STATUS=FAIL\" fi done echo \"CHANGES_MADE=NO\" echo \"SECRET_PRINTED=NO\" echo \"PRIVATE_ADDRESS_PRINTED=NO\" echo \"RESULT=OK\" echo \"BACKUP_OFFHOST_CURRENT_STATE=COMPLETE\" \n1599836 /usr/bin/bash -c set -eu test \"$(hostname -s)\" = \"pve01\" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W=\"$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)\" trap \"rm -rf -- \\\"$W\\\"\" EXIT BASE=\"https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history\" curl -fsSL \\ \"$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json\" \\ -o \"$W/204.json\" jq -e \" .command_id == \\\"HOMELAB-TAILS-FINAL-CLOSEOUT-204\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \\\"\\\") | contains(\\\"ALL_TAILS_CLOSED=YES\\\")) and ((.output // \\\"\\\") | contains(\\\"TAILS_FINAL_CLOSEOUT=COMPLETE\\\")) \" \"$W/204.json\" >/dev/null curl -fsSL \\ \"$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json\" \\ -o \"$W/122.json\" jq -e \" .command_id == \\\"HOMELAB-INDEPENDENT-FINAL-STATE-122\\\" and .status == \\\"OK\\\" and ((.rc | tonumber) == 0) and ((.output // \\\"\\\") | contains(\\\"HOMELAB_FINAL_STATE=BLOCKED\\\")) and ((.output // \\\"\\\") | contains(\\\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\\\")) \" \"$W/122.json\" >/dev/null echo \"PREVIOUS_RESULT_204=VERIFIED_OK\" echo \"HISTORICAL_BACKUP_BASELINE_122=VERIFIED\" echo \"REFERENCE_CHECK=OK\" echo \"ERROR_REGISTER_CHECK=OK\" echo \"AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)\" echo \"=== CURRENT_REFERENCE_STATE ===\" STATE=\"/etc/homelab/reference/state/current.json\" if test -s \"$STATE\" && jq -e \".\" \"$STATE\" >/dev/null; then jq -c \"{status,next_action,updated_at}\" \"$STATE\" else echo \"REFERENCE_STATE=INVALID_OR_MISSING\" fi echo \"=== CLUSTER_NODES ===\" pvesh get /nodes --output-format json | jq -c \"sort_by(.node)[] | {node,status,uptime}\" echo \"=== PVE_BACKUP_JOBS ===\" pvesh get /cluster/backup --output-format json | jq -c \"sort_by(.id)\" echo \"=== STORAGE_STATUS ===\" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo \"========== NODE=$NODE ==========\" if ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" true then echo \"SSH_STATUS=OK\" ssh \"${SSH_OPTS[@]}\" root@\"$NODE\" \" set -u echo HOST=\\$(hostname -s) echo UTC_NOW=\\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \\ homelab-backup-matrix-refresh.service \\ homelab-backup-matrix-refresh.timer \\ homelab-vm160-remote-backup.service \\ homelab-vm160-remote-backup.timer \\ homelab-cluster-admin-pull.service \\ homelab-cluster-admin-pull.timer \\ homelab-vm-local-dumps-2cloud.service \\ homelab-vm-local-dumps-2cloud.timer \\ homelab-mail-cloud-critical-upload.service \\ homelab-mail-cloud-critical-upload.timer \\ homelab-mail-cloud-edge-vm-upload.service \\ homelab-mail-cloud-edge-vm-upload.timer \\ homelab-mail-cloud-nextcloud-vm-upload.service \\ homelab-mail-cloud-nextcloud-vm-upload.timer \\ homelab-mail-cloud-restore-drill.service \\ homelab-mail-cloud-restore-drill.timer \\ homelab-verified-backup-health.service \\ homelab-verified-backup-health.timer \\ homelab-backup-sla-health.service \\ homelab-backup-sla-health.timer do if systemctl cat \\\"\\$U\\\" >/dev/null 2>&1 then systemctl show \\\"\\$U\\\" \\ -p Id \\ -p LoadState \\ -p ActiveState \\ -p SubState \\ -p Result \\ -p ExecMainStatus \\ -p UnitFileState \\ -p ActiveEnterTimestamp \\ -p InactiveEnterTimestamp \\ -p ExecStart \\ --no-pager else echo UNIT_MISSING=\\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \\\"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\\\" || true echo === TARGET_SCRIPT_STATE === for F in \\ /usr/local/sbin/homelab-backup-matrix-refresh \\ /usr/local/sbin/homelab-vm160-remote-backup \\ /usr/local/sbin/homelab-cluster-admin-pull \\ /usr/local/sbin/homelab-vm-local-dumps-2cloud \\ /usr/local/sbin/homelab-mail-cloud-critical-upload \\ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \\ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \\\"\\$F\\\" then stat -c \\\"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\\\" \\\"\\$F\\\" if /usr/bin/bash -n \\\"\\$F\\\" 2>/dev/null then echo SCRIPT_SYNTAX=\\$F:OK else echo SCRIPT_SYNTAX=\\$F:FAIL fi else echo FILE_MISSING=\\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \\ /var/lib/homelab-health/unified-backup-matrix.txt \\ /var/lib/homelab-health/backup-framework.txt \\ /var/lib/homelab-health/backup-sla.txt \\ /var/lib/homelab-health/backup-coverage-matrix.txt \\ /var/lib/homelab-health/vm-local-dumps-cloud.txt \\ /var/lib/homelab-health/vm-local-dumps-retention.txt \\ /var/lib/homelab-health/vm-backup.txt \\ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \\ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \\ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \\\"\\$H\\\" then echo HEALTH_FILE=\\$H stat -c \\\"HEALTH_MTIME=%y HEALTH_SIZE=%s\\\" \\\"\\$H\\\" sed -n \\\"1p\\\" \\\"\\$H\\\" else echo HEALTH_FILE_MISSING=\\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \\\"[v]zdump\\\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf x \\ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \\ -type f \\ \\( -name \\\"vzdump-*.zst\\\" \\ -o -name \\\"vzdump-*.lzo\\\" \\ -o -name \\\"vzdump-*.gz\\\" \\) \\ -printf \\\"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\\\" \\ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \\ -u homelab-backup-matrix-refresh.service \\ -u homelab-vm160-remote-backup.service \\ -u homelab-cluster-admin-pull.service \\ -n 120 \\ --no-pager \\ -o short-iso \\ 2>/dev/null || true \" else echo \"SSH_STATUS=FAIL\" fi done echo \"CHANGES_MADE=NO\" echo \"SECRET_PRINTED=NO\" echo \"PRIVATE_ADDRESS_PRINTED=NO\" echo \"RESULT=OK\" echo \"BACKUP_OFFHOST_CURRENT_STATE=COMPLETE\" \n1603200 ssh -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes root@pve01 set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat \"$U\" >/dev/null 2>&1 then systemctl show \"$U\" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\" if /usr/bin/bash -n \"$F\" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"$H\" then echo HEALTH_FILE=$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\" sed -n \"1p\" \"$H\" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true \n1603210 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat \"$U\" >/dev/null 2>&1 then systemctl show \"$U\" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\" if /usr/bin/bash -n \"$F\" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"$H\" then echo HEALTH_FILE=$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\" sed -n \"1p\" \"$H\" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true \n=== LOCAL_ARCHIVES ===\nARCHIVE_COUNT=12\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120\n=== TARGET_UNIT_LOGS ===\n2026-07-22T05:48:02+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T05:48:03+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T06:11:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T06:11:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T06:40:29+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T06:40:30+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T07:02:40+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T07:02:41+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T07:35:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T07:35:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:10:42+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:10:43+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:37:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:40:15+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:40:16+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T08:53:36+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T08:53:37+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_STATUS=FAIL\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json\n2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered\n2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE\n2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.\n2026-07-22T10:11:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.\n========== NODE=pve02 ==========\nSSH_STATUS=OK\nHOST=pve02\nUTC_NOW=2026-07-22T14:53:54Z\n=== FAILED_UNITS ===\nFAILED_UNIT_COUNT=0\n=== TARGET_UNITS ===\nUNIT_MISSING=homelab-backup-matrix-refresh.service\nUNIT_MISSING=homelab-backup-matrix-refresh.timer\nUNIT_MISSING=homelab-vm160-remote-backup.service\nUNIT_MISSING=homelab-vm160-remote-backup.timer\nUNIT_MISSING=homelab-cluster-admin-pull.service\nUNIT_MISSING=homelab-cluster-admin-pull.timer\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.service\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.timer\nUNIT_MISSING=homelab-mail-cloud-critical-upload.service\nUNIT_MISSING=homelab-mail-cloud-critical-upload.timer\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-restore-drill.service\nUNIT_MISSING=homelab-mail-cloud-restore-drill.timer\nUNIT_MISSING=homelab-verified-backup-health.service\nUNIT_MISSING=homelab-verified-backup-health.timer\nUNIT_MISSING=homelab-backup-sla-health.service\nUNIT_MISSING=homelab-backup-sla-health.timer\n=== RELEVANT_TIMERS ===\n=== TARGET_SCRIPT_STATE ===\nFILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh\nFILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup\nFILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull\nFILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill\n=== CURRENT_BACKUP_HEALTH ===\nHEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt\n=== RCLONE_REMOTE_NAMES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== ACTIVE_VZDUMP ===\n3930634 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat \"$U\" >/dev/null 2>&1 then systemctl show \"$U\" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\" if /usr/bin/bash -n \"$F\" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"$H\" then echo HEALTH_FILE=$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\" sed -n \"1p\" \"$H\" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true \n=== LOCAL_ARCHIVES ===\nARCHIVE_COUNT=10\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst SIZE=284443608 MTIME=2026-07-19T03:47:41.8525503200\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst SIZE=286538526 MTIME=2026-07-20T03:30:18.4386603270\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst SIZE=288349157 MTIME=2026-07-21T03:30:21.2480085730\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_22-03_30_01.tar.zst SIZE=290059742 MTIME=2026-07-22T03:30:22.8748248510\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst SIZE=220049041 MTIME=2026-07-19T03:47:58.9256407960\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst SIZE=220065825 MTIME=2026-07-20T03:30:30.8287264090\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst SIZE=220080970 MTIME=2026-07-21T03:30:33.3910787180\nARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_22-03_30_23.tar.zst SIZE=220095608 MTIME=2026-07-22T03:30:40.6809207660\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst SIZE=868048369 MTIME=2026-06-30T03:31:08.4903256780\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst SIZE=997503422 MTIME=2026-07-01T03:31:11.1700059310\n=== TARGET_UNIT_LOGS ===\n-- No entries --\n========== NODE=pve03 ==========\nSSH_STATUS=OK\nHOST=pve03\nUTC_NOW=2026-07-22T14:53:55Z\n=== FAILED_UNITS ===\nFAILED_UNIT_COUNT=1\nhomelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image\n=== TARGET_UNITS ===\nUNIT_MISSING=homelab-backup-matrix-refresh.service\nUNIT_MISSING=homelab-backup-matrix-refresh.timer\nUNIT_MISSING=homelab-vm160-remote-backup.service\nUNIT_MISSING=homelab-vm160-remote-backup.timer\nUNIT_MISSING=homelab-cluster-admin-pull.service\nUNIT_MISSING=homelab-cluster-admin-pull.timer\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.service\nUNIT_MISSING=homelab-vm-local-dumps-2cloud.timer\nUNIT_MISSING=homelab-mail-cloud-critical-upload.service\nUNIT_MISSING=homelab-mail-cloud-critical-upload.timer\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service\nUNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer\nUNIT_MISSING=homelab-mail-cloud-restore-drill.service\nUNIT_MISSING=homelab-mail-cloud-restore-drill.timer\nUNIT_MISSING=homelab-verified-backup-health.service\nUNIT_MISSING=homelab-verified-backup-health.timer\nUNIT_MISSING=homelab-backup-sla-health.service\nUNIT_MISSING=homelab-backup-sla-health.timer\n=== RELEVANT_TIMERS ===\n=== TARGET_SCRIPT_STATE ===\nFILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh\nFILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup\nFILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull\nFILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload\nFILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill\n=== CURRENT_BACKUP_HEALTH ===\nHEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt\nHEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt\n=== RCLONE_REMOTE_NAMES ===\npve01-mail-01:\npve01-mail-01-crypt:\npve01-mail-02:\npve01-mail-02-crypt:\npve01-mail-03:\npve01-mail-03-crypt:\npve01-mail-04:\npve01-mail-04-crypt:\npve01-mail-05:\npve01-mail-05-crypt:\npve02-mail-01:\npve02-mail-01-crypt:\npve02-mail-02:\npve02-mail-02-crypt:\npve02-mail-03:\npve02-mail-03-crypt:\npve02-mail-04:\npve02-mail-04-crypt:\npve03-mail-01:\npve03-mail-01-crypt:\npve03-mail-02:\npve03-mail-02-crypt:\npve03-mail-03:\npve03-mail-03-crypt:\npve03-mail-04:\npve03-mail-04-crypt:\n=== ACTIVE_VZDUMP ===\n2313989 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat \"$U\" >/dev/null 2>&1 then systemctl show \"$U\" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"$F\" if /usr/bin/bash -n \"$F\" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"$H\" then echo HEALTH_FILE=$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"$H\" sed -n \"1p\" \"$H\" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \\( -name \"vzdump-*.zst\" -o -name \"vzdump-*.lzo\" -o -name \"vzdump-*.gz\" \\) -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\\n\" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true \n=== LOCAL_ARCHIVES ===\nARCHIVE_COUNT=8\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst SIZE=11013723404 MTIME=2026-07-19T03:33:53.3463878040\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst SIZE=11116472459 MTIME=2026-07-20T03:33:53.2839064370\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst SIZE=11043834894 MTIME=2026-07-21T03:33:54.3725716230\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 MTIME=2026-07-22T03:33:58.8961071480\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst SIZE=838951976 MTIME=2026-07-19T03:34:33.1034859540\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst SIZE=838817925 MTIME=2026-07-20T03:34:37.1719936670\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst SIZE=1137072350 MTIME=2026-07-21T03:34:37.6276811360\nARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_22-03_33_59.vma.zst SIZE=1177407443 MTIME=2026-07-22T03:34:44.3412293530\n=== TARGET_UNIT_LOGS ===\n-- No entries --\nCHANGES_MADE=NO\nSECRET_PRINTED=NO\nPRIVATE_ADDRESS_PRINTED=NO\nRESULT=OK\nBACKUP_OFFHOST_CURRENT_STATE=COMPLETE\n"
}
View Git Blame Copy Permalink