CHAT_OUTPUT_BEGIN COMMAND_ID=BACKUP-OFFHOST-CURRENT-STATE-212 STATUS=OK RC=0 HOST=pve01 MODE=read-only COMPONENT=backup-audit REFERENCE_REGISTER_CHECK=OK REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e ERROR_REGISTER_CHECK=OK ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752 COMMAND_SHA256=5b9f985ab6daceaad18e56d2da3dde0d2dba7eefdfe7f75e83eee6bef0a0ad8a DUPLICATE_FAILED_COMMAND_BLOCKED=false EXECUTION_STARTED=true CHANGES_MADE=false SANITIZED=yes SECRETS_INCLUDED=no PRIVATE_ADDRESSES_INCLUDED=no RAW_EVIDENCE_SHA256=9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2 SANITIZED_OUTPUT_SHA256=9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2 OUTPUT_BEGIN validation=PASS PREVIOUS_RESULT_204=VERIFIED_OK HISTORICAL_BACKUP_BASELINE_122=VERIFIED REFERENCE_CHECK=OK ERROR_REGISTER_CHECK=OK AUDIT_GENERATED_AT=2026-07-22T14:53:51Z === CURRENT_REFERENCE_STATE === {"status":"AUTO_REFRESH_READY","next_action":null,"updated_at":"2026-07-22T14:22:14+00:00"} === CLUSTER_NODES === {"node":"pve01","status":"online","uptime":540902} {"node":"pve02","status":"online","uptime":540921} {"node":"pve03","status":"online","uptime":540774} === PVE_BACKUP_JOBS === [{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784766600,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}] === STORAGE_STATUS === Name Type Status Total (KiB) Used (KiB) Available (KiB) % local dir active 102626232 42074800 55292168 41.00% local-lvm lvmthin active 848797696 305906689 542891006 36.04% ========== NODE=pve01 ========== SSH_STATUS=OK HOST=pve01 UTC_NOW=2026-07-22T14:53:53Z === FAILED_UNITS === FAILED_UNIT_COUNT=3 homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state homelab-cluster-admin-pull.service loaded failed failed Pull cluster-admin observer health into pve01 homelab-vm160-remote-backup.service loaded failed failed VM160 full-image backup streamed from pve02 to pve03 === TARGET_UNITS === Result=exit-code ExecMainStatus=1 ExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-backup-matrix-refresh.service LoadState=loaded ActiveState=failed SubState=failed UnitFileState=static ActiveEnterTimestamp= InactiveEnterTimestamp=Wed 2026-07-22 10:11:15 MSK Result=success Id=homelab-backup-matrix-refresh.timer LoadState=loaded ActiveState=inactive SubState=dead UnitFileState=disabled ActiveEnterTimestamp= InactiveEnterTimestamp= Result=exit-code ExecMainStatus=1 ExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-vm160-remote-backup.service LoadState=loaded ActiveState=failed SubState=failed UnitFileState=static ActiveEnterTimestamp= InactiveEnterTimestamp=Tue 2026-07-21 02:39:14 MSK Result=success Id=homelab-vm160-remote-backup.timer LoadState=loaded ActiveState=inactive SubState=dead UnitFileState=disabled ActiveEnterTimestamp= InactiveEnterTimestamp= Result=exit-code ExecMainStatus=1 ExecStart={ path=/usr/local/sbin/homelab-cluster-admin-pull ; argv[]=/usr/local/sbin/homelab-cluster-admin-pull ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-cluster-admin-pull.service LoadState=loaded ActiveState=failed SubState=failed UnitFileState=static ActiveEnterTimestamp= InactiveEnterTimestamp=Tue 2026-07-21 02:37:04 MSK Result=success Id=homelab-cluster-admin-pull.timer LoadState=loaded ActiveState=inactive SubState=dead UnitFileState=disabled ActiveEnterTimestamp= InactiveEnterTimestamp= Result=success ExecMainStatus=0 ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-vm-local-dumps-2cloud.service LoadState=loaded ActiveState=inactive SubState=dead UnitFileState=static ActiveEnterTimestamp= InactiveEnterTimestamp=Sun 2026-07-19 10:13:40 MSK Result=success Id=homelab-vm-local-dumps-2cloud.timer LoadState=loaded ActiveState=active SubState=waiting UnitFileState=enabled ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK InactiveEnterTimestamp= Result=success ExecMainStatus=0 ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-critical-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-critical-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-mail-cloud-critical-upload.service LoadState=loaded ActiveState=inactive SubState=dead UnitFileState=static ActiveEnterTimestamp= InactiveEnterTimestamp=Wed 2026-07-22 06:46:47 MSK Result=success Id=homelab-mail-cloud-critical-upload.timer LoadState=loaded ActiveState=active SubState=waiting UnitFileState=enabled ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK InactiveEnterTimestamp= Result=success ExecMainStatus=0 ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-mail-cloud-edge-vm-upload.service LoadState=loaded ActiveState=inactive SubState=dead UnitFileState=static ActiveEnterTimestamp= InactiveEnterTimestamp=Sun 2026-07-19 09:41:56 MSK Result=success Id=homelab-mail-cloud-edge-vm-upload.timer LoadState=loaded ActiveState=active SubState=waiting UnitFileState=enabled ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK InactiveEnterTimestamp= Result=success ExecMainStatus=0 ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-mail-cloud-nextcloud-vm-upload.service LoadState=loaded ActiveState=inactive SubState=dead UnitFileState=static ActiveEnterTimestamp= InactiveEnterTimestamp=Wed 2026-07-22 07:45:01 MSK Result=success Id=homelab-mail-cloud-nextcloud-vm-upload.timer LoadState=loaded ActiveState=active SubState=waiting UnitFileState=enabled ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK InactiveEnterTimestamp= Result=success ExecMainStatus=0 ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-mail-cloud-restore-drill.service LoadState=loaded ActiveState=inactive SubState=dead UnitFileState=static ActiveEnterTimestamp= InactiveEnterTimestamp=Mon 2026-07-20 07:25:14 MSK Result=success Id=homelab-mail-cloud-restore-drill.timer LoadState=loaded ActiveState=active SubState=waiting UnitFileState=enabled ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK InactiveEnterTimestamp= Result=success ExecMainStatus=0 ExecStart={ path=/usr/local/sbin/homelab-backup-health-evidence ; argv[]=/usr/local/sbin/homelab-backup-health-evidence ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-verified-backup-health.service LoadState=loaded ActiveState=inactive SubState=dead UnitFileState=static ActiveEnterTimestamp= InactiveEnterTimestamp=Wed 2026-07-22 06:40:29 MSK Result=success Id=homelab-verified-backup-health.timer LoadState=loaded ActiveState=active SubState=waiting UnitFileState=enabled ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK InactiveEnterTimestamp= Result=success ExecMainStatus=0 ExecStart={ path=/usr/local/sbin/homelab-backup-sla-health ; argv[]=/usr/local/sbin/homelab-backup-sla-health ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } Id=homelab-backup-sla-health.service LoadState=loaded ActiveState=inactive SubState=dead UnitFileState=static ActiveEnterTimestamp= InactiveEnterTimestamp=Wed 2026-07-22 08:10:42 MSK Result=success Id=homelab-backup-sla-health.timer LoadState=loaded ActiveState=active SubState=waiting UnitFileState=enabled ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK InactiveEnterTimestamp= === RELEVANT_TIMERS === Thu 2026-07-23 06:38:07 MSK 12h Wed 2026-07-22 06:39:05 MSK 11h ago homelab-verified-backup-health.timer homelab-verified-backup-health.service Thu 2026-07-23 06:44:08 MSK 12h Wed 2026-07-22 06:44:25 MSK 11h ago homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-critical-upload.service Thu 2026-07-23 07:58:35 MSK 14h Wed 2026-07-22 07:42:14 MSK 10h ago homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service Thu 2026-07-23 08:12:51 MSK 14h Wed 2026-07-22 08:10:41 MSK 9h ago homelab-backup-sla-health.timer homelab-backup-sla-health.service Sun 2026-07-26 09:46:53 MSK 3 days Sun 2026-07-19 09:32:29 MSK 3 days ago homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-edge-vm-upload.service Sun 2026-07-26 10:23:39 MSK 3 days Sun 2026-07-19 10:06:07 MSK 3 days ago homelab-vm-local-dumps-2cloud.timer homelab-vm-local-dumps-2cloud.service Mon 2026-07-27 07:34:27 MSK 4 days Mon 2026-07-20 07:25:14 MSK 2 days ago homelab-mail-cloud-restore-drill.timer homelab-mail-cloud-restore-drill.service === TARGET_SCRIPT_STATE === FILE=/usr/local/sbin/homelab-backup-matrix-refresh TYPE=regular file MODE=755 OWNER=root:root SIZE=8714 MTIME=2026-07-10 13:02:04.846868245 +0300 SCRIPT_SYNTAX=/usr/local/sbin/homelab-backup-matrix-refresh:FAIL FILE=/usr/local/sbin/homelab-vm160-remote-backup TYPE=regular file MODE=755 OWNER=root:root SIZE=3140 MTIME=2026-07-18 01:38:18.170629793 +0300 SCRIPT_SYNTAX=/usr/local/sbin/homelab-vm160-remote-backup:OK FILE=/usr/local/sbin/homelab-cluster-admin-pull TYPE=regular file MODE=755 OWNER=root:root SIZE=2501 MTIME=2026-07-13 01:15:06.431144817 +0300 SCRIPT_SYNTAX=/usr/local/sbin/homelab-cluster-admin-pull:OK FILE=/usr/local/sbin/homelab-vm-local-dumps-2cloud TYPE=regular file MODE=700 OWNER=root:root SIZE=7272 MTIME=2026-07-06 18:49:49.740645050 +0300 SCRIPT_SYNTAX=/usr/local/sbin/homelab-vm-local-dumps-2cloud:OK FILE=/usr/local/sbin/homelab-mail-cloud-critical-upload TYPE=regular file MODE=700 OWNER=root:root SIZE=7836 MTIME=2026-07-17 14:03:50.373696143 +0300 SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-critical-upload:OK FILE=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=12899 MTIME=2026-07-13 21:27:22.164609073 +0300 SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:OK FILE=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=6542 MTIME=2026-06-30 15:00:33.588620116 +0300 SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:OK FILE=/usr/local/sbin/homelab-mail-cloud-restore-drill TYPE=regular file MODE=700 OWNER=root:root SIZE=7995 MTIME=2026-07-10 04:14:10.054957455 +0300 SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-restore-drill:OK === CURRENT_BACKUP_HEALTH === HEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt HEALTH_FILE=/var/lib/homelab-health/backup-framework.txt HEALTH_MTIME=2026-07-22 10:11:15.588809944 +0300 HEALTH_SIZE=803 STATUS=OK HEALTH_FILE=/var/lib/homelab-health/backup-sla.txt HEALTH_MTIME=2026-07-22 17:45:04.576070554 +0300 HEALTH_SIZE=154 STATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md HEALTH_FILE=/var/lib/homelab-health/backup-coverage-matrix.txt HEALTH_MTIME=2026-07-22 10:11:15.236509358 +0300 HEALTH_SIZE=808 STATUS=OK HEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt HEALTH_MTIME=2026-07-22 10:11:15.694007276 +0300 HEALTH_SIZE=800 STATUS=OK HEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-retention.txt HEALTH_MTIME=2026-07-22 10:11:15.711074323 +0300 HEALTH_SIZE=826 STATUS=OK HEALTH_FILE=/var/lib/homelab-health/vm-backup.txt HEALTH_MTIME=2026-07-22 07:35:14.956502473 +0300 HEALTH_SIZE=170 STATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active HEALTH_FILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt HEALTH_MTIME=2026-07-09 08:07:47.213452046 +0300 HEALTH_SIZE=217 STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md HEALTH_FILE=/var/lib/homelab-health/edge-vm-offhost-freshness.txt HEALTH_MTIME=2026-07-22 04:46:23.189237991 +0300 HEALTH_SIZE=409 STATUS=OK TS=2026-07-22T01:39:30Z TYPE=edge-vm-offhost-freshness BAD=0 VMID=130 SOURCE_NODE=pve03 DEST_NODE=pve02 SRC=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst DEST=/mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 SHA256=a8e5179614e02e6933617adb30cb95a35a1b5d17b30c9f00a2626cfa06b14710 COPY_STATUS=copied ZSTD_RC=0 VMA_RC=0 KEEP=4 HEALTH_FILE=/var/lib/homelab-health/mail-cloud-restore-drill.txt HEALTH_MTIME=2026-07-22 10:11:15.491775958 +0300 HEALTH_SIZE=830 STATUS=STALE === RCLONE_REMOTE_NAMES === pve01-mail-01: pve01-mail-01-crypt: pve01-mail-02: pve01-mail-02-crypt: pve01-mail-03: pve01-mail-03-crypt: pve01-mail-04: pve01-mail-04-crypt: pve01-mail-05: pve01-mail-05-crypt: pve02-mail-01: pve02-mail-01-crypt: pve02-mail-02: pve02-mail-02-crypt: pve02-mail-03: pve02-mail-03-crypt: pve02-mail-04: pve02-mail-04-crypt: pve03-mail-01: pve03-mail-01-crypt: pve03-mail-02: pve03-mail-02-crypt: pve03-mail-03: pve03-mail-03-crypt: pve03-mail-04: pve03-mail-04-crypt: === ACTIVE_VZDUMP === 1599768 bash /usr/local/sbin/homelab-chat-run BACKUP-OFFHOST-CURRENT-STATE-212 read-only backup-audit -- /usr/bin/bash -c set -eu test "$(hostname -s)" = "pve01" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W="$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)" trap "rm -rf -- \"$W\"" EXIT BASE="https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history" curl -fsSL \ "$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json" \ -o "$W/204.json" jq -e " .command_id == \"HOMELAB-TAILS-FINAL-CLOSEOUT-204\" and .status == \"OK\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \"\") | contains(\"ALL_TAILS_CLOSED=YES\")) and ((.output // \"\") | contains(\"TAILS_FINAL_CLOSEOUT=COMPLETE\")) " "$W/204.json" >/dev/null curl -fsSL \ "$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json" \ -o "$W/122.json" jq -e " .command_id == \"HOMELAB-INDEPENDENT-FINAL-STATE-122\" and .status == \"OK\" and ((.rc | tonumber) == 0) and ((.output // \"\") | contains(\"HOMELAB_FINAL_STATE=BLOCKED\")) and ((.output // \"\") | contains(\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\")) " "$W/122.json" >/dev/null echo "PREVIOUS_RESULT_204=VERIFIED_OK" echo "HISTORICAL_BACKUP_BASELINE_122=VERIFIED" echo "REFERENCE_CHECK=OK" echo "ERROR_REGISTER_CHECK=OK" echo "AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "=== CURRENT_REFERENCE_STATE ===" STATE="/etc/homelab/reference/state/current.json" if test -s "$STATE" && jq -e "." "$STATE" >/dev/null; then jq -c "{status,next_action,updated_at}" "$STATE" else echo "REFERENCE_STATE=INVALID_OR_MISSING" fi echo "=== CLUSTER_NODES ===" pvesh get /nodes --output-format json | jq -c "sort_by(.node)[] | {node,status,uptime}" echo "=== PVE_BACKUP_JOBS ===" pvesh get /cluster/backup --output-format json | jq -c "sort_by(.id)" echo "=== STORAGE_STATUS ===" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo "========== NODE=$NODE ==========" if ssh "${SSH_OPTS[@]}" root@"$NODE" true then echo "SSH_STATUS=OK" ssh "${SSH_OPTS[@]}" root@"$NODE" " set -u echo HOST=\$(hostname -s) echo UTC_NOW=\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \ homelab-backup-matrix-refresh.service \ homelab-backup-matrix-refresh.timer \ homelab-vm160-remote-backup.service \ homelab-vm160-remote-backup.timer \ homelab-cluster-admin-pull.service \ homelab-cluster-admin-pull.timer \ homelab-vm-local-dumps-2cloud.service \ homelab-vm-local-dumps-2cloud.timer \ homelab-mail-cloud-critical-upload.service \ homelab-mail-cloud-critical-upload.timer \ homelab-mail-cloud-edge-vm-upload.service \ homelab-mail-cloud-edge-vm-upload.timer \ homelab-mail-cloud-nextcloud-vm-upload.service \ homelab-mail-cloud-nextcloud-vm-upload.timer \ homelab-mail-cloud-restore-drill.service \ homelab-mail-cloud-restore-drill.timer \ homelab-verified-backup-health.service \ homelab-verified-backup-health.timer \ homelab-backup-sla-health.service \ homelab-backup-sla-health.timer do if systemctl cat \"\$U\" >/dev/null 2>&1 then systemctl show \"\$U\" \ -p Id \ -p LoadState \ -p ActiveState \ -p SubState \ -p Result \ -p ExecMainStatus \ -p UnitFileState \ -p ActiveEnterTimestamp \ -p InactiveEnterTimestamp \ -p ExecStart \ --no-pager else echo UNIT_MISSING=\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in \ /usr/local/sbin/homelab-backup-matrix-refresh \ /usr/local/sbin/homelab-vm160-remote-backup \ /usr/local/sbin/homelab-cluster-admin-pull \ /usr/local/sbin/homelab-vm-local-dumps-2cloud \ /usr/local/sbin/homelab-mail-cloud-critical-upload \ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"\$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"\$F\" if /usr/bin/bash -n \"\$F\" 2>/dev/null then echo SCRIPT_SYNTAX=\$F:OK else echo SCRIPT_SYNTAX=\$F:FAIL fi else echo FILE_MISSING=\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \ /var/lib/homelab-health/unified-backup-matrix.txt \ /var/lib/homelab-health/backup-framework.txt \ /var/lib/homelab-health/backup-sla.txt \ /var/lib/homelab-health/backup-coverage-matrix.txt \ /var/lib/homelab-health/vm-local-dumps-cloud.txt \ /var/lib/homelab-health/vm-local-dumps-retention.txt \ /var/lib/homelab-health/vm-backup.txt \ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"\$H\" then echo HEALTH_FILE=\$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"\$H\" sed -n \"1p\" \"\$H\" else echo HEALTH_FILE_MISSING=\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf x \ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n\" \ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \ -u homelab-backup-matrix-refresh.service \ -u homelab-vm160-remote-backup.service \ -u homelab-cluster-admin-pull.service \ -n 120 \ --no-pager \ -o short-iso \ 2>/dev/null || true " else echo "SSH_STATUS=FAIL" fi done echo "CHANGES_MADE=NO" echo "SECRET_PRINTED=NO" echo "PRIVATE_ADDRESS_PRINTED=NO" echo "RESULT=OK" echo "BACKUP_OFFHOST_CURRENT_STATE=COMPLETE" 1599834 timeout --foreground --preserve-status 300 /usr/bin/bash -c set -eu test "$(hostname -s)" = "pve01" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W="$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)" trap "rm -rf -- \"$W\"" EXIT BASE="https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history" curl -fsSL \ "$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json" \ -o "$W/204.json" jq -e " .command_id == \"HOMELAB-TAILS-FINAL-CLOSEOUT-204\" and .status == \"OK\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \"\") | contains(\"ALL_TAILS_CLOSED=YES\")) and ((.output // \"\") | contains(\"TAILS_FINAL_CLOSEOUT=COMPLETE\")) " "$W/204.json" >/dev/null curl -fsSL \ "$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json" \ -o "$W/122.json" jq -e " .command_id == \"HOMELAB-INDEPENDENT-FINAL-STATE-122\" and .status == \"OK\" and ((.rc | tonumber) == 0) and ((.output // \"\") | contains(\"HOMELAB_FINAL_STATE=BLOCKED\")) and ((.output // \"\") | contains(\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\")) " "$W/122.json" >/dev/null echo "PREVIOUS_RESULT_204=VERIFIED_OK" echo "HISTORICAL_BACKUP_BASELINE_122=VERIFIED" echo "REFERENCE_CHECK=OK" echo "ERROR_REGISTER_CHECK=OK" echo "AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "=== CURRENT_REFERENCE_STATE ===" STATE="/etc/homelab/reference/state/current.json" if test -s "$STATE" && jq -e "." "$STATE" >/dev/null; then jq -c "{status,next_action,updated_at}" "$STATE" else echo "REFERENCE_STATE=INVALID_OR_MISSING" fi echo "=== CLUSTER_NODES ===" pvesh get /nodes --output-format json | jq -c "sort_by(.node)[] | {node,status,uptime}" echo "=== PVE_BACKUP_JOBS ===" pvesh get /cluster/backup --output-format json | jq -c "sort_by(.id)" echo "=== STORAGE_STATUS ===" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo "========== NODE=$NODE ==========" if ssh "${SSH_OPTS[@]}" root@"$NODE" true then echo "SSH_STATUS=OK" ssh "${SSH_OPTS[@]}" root@"$NODE" " set -u echo HOST=\$(hostname -s) echo UTC_NOW=\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \ homelab-backup-matrix-refresh.service \ homelab-backup-matrix-refresh.timer \ homelab-vm160-remote-backup.service \ homelab-vm160-remote-backup.timer \ homelab-cluster-admin-pull.service \ homelab-cluster-admin-pull.timer \ homelab-vm-local-dumps-2cloud.service \ homelab-vm-local-dumps-2cloud.timer \ homelab-mail-cloud-critical-upload.service \ homelab-mail-cloud-critical-upload.timer \ homelab-mail-cloud-edge-vm-upload.service \ homelab-mail-cloud-edge-vm-upload.timer \ homelab-mail-cloud-nextcloud-vm-upload.service \ homelab-mail-cloud-nextcloud-vm-upload.timer \ homelab-mail-cloud-restore-drill.service \ homelab-mail-cloud-restore-drill.timer \ homelab-verified-backup-health.service \ homelab-verified-backup-health.timer \ homelab-backup-sla-health.service \ homelab-backup-sla-health.timer do if systemctl cat \"\$U\" >/dev/null 2>&1 then systemctl show \"\$U\" \ -p Id \ -p LoadState \ -p ActiveState \ -p SubState \ -p Result \ -p ExecMainStatus \ -p UnitFileState \ -p ActiveEnterTimestamp \ -p InactiveEnterTimestamp \ -p ExecStart \ --no-pager else echo UNIT_MISSING=\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in \ /usr/local/sbin/homelab-backup-matrix-refresh \ /usr/local/sbin/homelab-vm160-remote-backup \ /usr/local/sbin/homelab-cluster-admin-pull \ /usr/local/sbin/homelab-vm-local-dumps-2cloud \ /usr/local/sbin/homelab-mail-cloud-critical-upload \ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"\$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"\$F\" if /usr/bin/bash -n \"\$F\" 2>/dev/null then echo SCRIPT_SYNTAX=\$F:OK else echo SCRIPT_SYNTAX=\$F:FAIL fi else echo FILE_MISSING=\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \ /var/lib/homelab-health/unified-backup-matrix.txt \ /var/lib/homelab-health/backup-framework.txt \ /var/lib/homelab-health/backup-sla.txt \ /var/lib/homelab-health/backup-coverage-matrix.txt \ /var/lib/homelab-health/vm-local-dumps-cloud.txt \ /var/lib/homelab-health/vm-local-dumps-retention.txt \ /var/lib/homelab-health/vm-backup.txt \ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"\$H\" then echo HEALTH_FILE=\$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"\$H\" sed -n \"1p\" \"\$H\" else echo HEALTH_FILE_MISSING=\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf x \ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n\" \ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \ -u homelab-backup-matrix-refresh.service \ -u homelab-vm160-remote-backup.service \ -u homelab-cluster-admin-pull.service \ -n 120 \ --no-pager \ -o short-iso \ 2>/dev/null || true " else echo "SSH_STATUS=FAIL" fi done echo "CHANGES_MADE=NO" echo "SECRET_PRINTED=NO" echo "PRIVATE_ADDRESS_PRINTED=NO" echo "RESULT=OK" echo "BACKUP_OFFHOST_CURRENT_STATE=COMPLETE" 1599836 /usr/bin/bash -c set -eu test "$(hostname -s)" = "pve01" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null /usr/local/sbin/homelab-reference validate W="$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)" trap "rm -rf -- \"$W\"" EXIT BASE="https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history" curl -fsSL \ "$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json" \ -o "$W/204.json" jq -e " .command_id == \"HOMELAB-TAILS-FINAL-CLOSEOUT-204\" and .status == \"OK\" and ((.rc | tonumber) == 0) and .reference_register_checked and .error_register_checked and ((.output // \"\") | contains(\"ALL_TAILS_CLOSED=YES\")) and ((.output // \"\") | contains(\"TAILS_FINAL_CLOSEOUT=COMPLETE\")) " "$W/204.json" >/dev/null curl -fsSL \ "$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json" \ -o "$W/122.json" jq -e " .command_id == \"HOMELAB-INDEPENDENT-FINAL-STATE-122\" and .status == \"OK\" and ((.rc | tonumber) == 0) and ((.output // \"\") | contains(\"HOMELAB_FINAL_STATE=BLOCKED\")) and ((.output // \"\") | contains(\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\")) " "$W/122.json" >/dev/null echo "PREVIOUS_RESULT_204=VERIFIED_OK" echo "HISTORICAL_BACKUP_BASELINE_122=VERIFIED" echo "REFERENCE_CHECK=OK" echo "ERROR_REGISTER_CHECK=OK" echo "AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "=== CURRENT_REFERENCE_STATE ===" STATE="/etc/homelab/reference/state/current.json" if test -s "$STATE" && jq -e "." "$STATE" >/dev/null; then jq -c "{status,next_action,updated_at}" "$STATE" else echo "REFERENCE_STATE=INVALID_OR_MISSING" fi echo "=== CLUSTER_NODES ===" pvesh get /nodes --output-format json | jq -c "sort_by(.node)[] | {node,status,uptime}" echo "=== PVE_BACKUP_JOBS ===" pvesh get /cluster/backup --output-format json | jq -c "sort_by(.id)" echo "=== STORAGE_STATUS ===" pvesm status || true SSH_OPTS=( -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes ) for NODE in pve01 pve02 pve03 do echo "========== NODE=$NODE ==========" if ssh "${SSH_OPTS[@]}" root@"$NODE" true then echo "SSH_STATUS=OK" ssh "${SSH_OPTS[@]}" root@"$NODE" " set -u echo HOST=\$(hostname -s) echo UTC_NOW=\$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in \ homelab-backup-matrix-refresh.service \ homelab-backup-matrix-refresh.timer \ homelab-vm160-remote-backup.service \ homelab-vm160-remote-backup.timer \ homelab-cluster-admin-pull.service \ homelab-cluster-admin-pull.timer \ homelab-vm-local-dumps-2cloud.service \ homelab-vm-local-dumps-2cloud.timer \ homelab-mail-cloud-critical-upload.service \ homelab-mail-cloud-critical-upload.timer \ homelab-mail-cloud-edge-vm-upload.service \ homelab-mail-cloud-edge-vm-upload.timer \ homelab-mail-cloud-nextcloud-vm-upload.service \ homelab-mail-cloud-nextcloud-vm-upload.timer \ homelab-mail-cloud-restore-drill.service \ homelab-mail-cloud-restore-drill.timer \ homelab-verified-backup-health.service \ homelab-verified-backup-health.timer \ homelab-backup-sla-health.service \ homelab-backup-sla-health.timer do if systemctl cat \"\$U\" >/dev/null 2>&1 then systemctl show \"\$U\" \ -p Id \ -p LoadState \ -p ActiveState \ -p SubState \ -p Result \ -p ExecMainStatus \ -p UnitFileState \ -p ActiveEnterTimestamp \ -p InactiveEnterTimestamp \ -p ExecStart \ --no-pager else echo UNIT_MISSING=\$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" || true echo === TARGET_SCRIPT_STATE === for F in \ /usr/local/sbin/homelab-backup-matrix-refresh \ /usr/local/sbin/homelab-vm160-remote-backup \ /usr/local/sbin/homelab-cluster-admin-pull \ /usr/local/sbin/homelab-vm-local-dumps-2cloud \ /usr/local/sbin/homelab-mail-cloud-critical-upload \ /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \ /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \ /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e \"\$F\" then stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"\$F\" if /usr/bin/bash -n \"\$F\" 2>/dev/null then echo SCRIPT_SYNTAX=\$F:OK else echo SCRIPT_SYNTAX=\$F:FAIL fi else echo FILE_MISSING=\$F fi done echo === CURRENT_BACKUP_HEALTH === for H in \ /var/lib/homelab-health/unified-backup-matrix.txt \ /var/lib/homelab-health/backup-framework.txt \ /var/lib/homelab-health/backup-sla.txt \ /var/lib/homelab-health/backup-coverage-matrix.txt \ /var/lib/homelab-health/vm-local-dumps-cloud.txt \ /var/lib/homelab-health/vm-local-dumps-retention.txt \ /var/lib/homelab-health/vm-backup.txt \ /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \ /var/lib/homelab-health/edge-vm-offhost-freshness.txt \ /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f \"\$H\" then echo HEALTH_FILE=\$H stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"\$H\" sed -n \"1p\" \"\$H\" else echo HEALTH_FILE_MISSING=\$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf x \ 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve \ -type f \ \( -name \"vzdump-*.zst\" \ -o -name \"vzdump-*.lzo\" \ -o -name \"vzdump-*.gz\" \) \ -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n\" \ 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl \ -u homelab-backup-matrix-refresh.service \ -u homelab-vm160-remote-backup.service \ -u homelab-cluster-admin-pull.service \ -n 120 \ --no-pager \ -o short-iso \ 2>/dev/null || true " else echo "SSH_STATUS=FAIL" fi done echo "CHANGES_MADE=NO" echo "SECRET_PRINTED=NO" echo "PRIVATE_ADDRESS_PRINTED=NO" echo "RESULT=OK" echo "BACKUP_OFFHOST_CURRENT_STATE=COMPLETE" 1603200 ssh -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes root@pve01 set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat "$U" >/dev/null 2>&1 then systemctl show "$U" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e "$F" then stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F" if /usr/bin/bash -n "$F" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f "$H" then echo HEALTH_FILE=$H stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H" sed -n "1p" "$H" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true 1603210 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat "$U" >/dev/null 2>&1 then systemctl show "$U" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e "$F" then stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F" if /usr/bin/bash -n "$F" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f "$H" then echo HEALTH_FILE=$H stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H" sed -n "1p" "$H" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true === LOCAL_ARCHIVES === ARCHIVE_COUNT=12 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120 === TARGET_UNIT_LOGS === 2026-07-22T05:48:02+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... 2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_STATUS=FAIL 2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_TOTAL=41 2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_OK=30 2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_BAD=6 2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_UNMAPPED=0 2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_DISABLED=5 2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json 2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered 2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE 2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. 2026-07-22T05:48:03+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. 2026-07-22T06:11:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... 2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_STATUS=FAIL 2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_TOTAL=41 2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_OK=30 2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_BAD=6 2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_UNMAPPED=0 2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_DISABLED=5 2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json 2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered 2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE 2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. 2026-07-22T06:11:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. 2026-07-22T06:40:29+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... 2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_STATUS=FAIL 2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_TOTAL=41 2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_OK=30 2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_BAD=6 2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_UNMAPPED=0 2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_DISABLED=5 2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json 2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered 2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE 2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. 2026-07-22T06:40:30+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. 2026-07-22T07:02:40+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... 2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_STATUS=FAIL 2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_TOTAL=41 2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_OK=30 2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_BAD=6 2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_UNMAPPED=0 2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_DISABLED=5 2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json 2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered 2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE 2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. 2026-07-22T07:02:41+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. 2026-07-22T07:35:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... 2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_STATUS=FAIL 2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_TOTAL=41 2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_OK=30 2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_BAD=6 2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_UNMAPPED=0 2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_DISABLED=5 2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json 2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered 2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE 2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. 2026-07-22T07:35:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. 2026-07-22T08:10:42+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... 2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_STATUS=FAIL 2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_TOTAL=41 2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_OK=30 2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_BAD=6 2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_UNMAPPED=0 2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_DISABLED=5 2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json 2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered 2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE 2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. 2026-07-22T08:10:43+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. 2026-07-22T08:37:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... 2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_STATUS=FAIL 2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_TOTAL=41 2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_OK=30 2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_BAD=6 2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_UNMAPPED=0 2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_DISABLED=5 2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json 2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered 2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE 2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. 2026-07-22T08:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. 2026-07-22T08:40:15+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... 2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_STATUS=FAIL 2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_TOTAL=41 2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_OK=30 2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_BAD=6 2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_UNMAPPED=0 2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_DISABLED=5 2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json 2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered 2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE 2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. 2026-07-22T08:40:16+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. 2026-07-22T08:53:36+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... 2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_STATUS=FAIL 2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_TOTAL=41 2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_OK=30 2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_BAD=6 2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_UNMAPPED=0 2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_DISABLED=5 2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json 2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered 2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE 2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. 2026-07-22T08:53:37+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. 2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state... 2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_STATUS=FAIL 2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41 2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30 2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6 2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0 2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5 2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json 2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered 2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE 2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'. 2026-07-22T10:11:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state. ========== NODE=pve02 ========== SSH_STATUS=OK HOST=pve02 UTC_NOW=2026-07-22T14:53:54Z === FAILED_UNITS === FAILED_UNIT_COUNT=0 === TARGET_UNITS === UNIT_MISSING=homelab-backup-matrix-refresh.service UNIT_MISSING=homelab-backup-matrix-refresh.timer UNIT_MISSING=homelab-vm160-remote-backup.service UNIT_MISSING=homelab-vm160-remote-backup.timer UNIT_MISSING=homelab-cluster-admin-pull.service UNIT_MISSING=homelab-cluster-admin-pull.timer UNIT_MISSING=homelab-vm-local-dumps-2cloud.service UNIT_MISSING=homelab-vm-local-dumps-2cloud.timer UNIT_MISSING=homelab-mail-cloud-critical-upload.service UNIT_MISSING=homelab-mail-cloud-critical-upload.timer UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer UNIT_MISSING=homelab-mail-cloud-restore-drill.service UNIT_MISSING=homelab-mail-cloud-restore-drill.timer UNIT_MISSING=homelab-verified-backup-health.service UNIT_MISSING=homelab-verified-backup-health.timer UNIT_MISSING=homelab-backup-sla-health.service UNIT_MISSING=homelab-backup-sla-health.timer === RELEVANT_TIMERS === === TARGET_SCRIPT_STATE === FILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh FILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup FILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull FILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill === CURRENT_BACKUP_HEALTH === HEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt === RCLONE_REMOTE_NAMES === pve01-mail-01: pve01-mail-01-crypt: pve01-mail-02: pve01-mail-02-crypt: pve01-mail-03: pve01-mail-03-crypt: pve01-mail-04: pve01-mail-04-crypt: pve01-mail-05: pve01-mail-05-crypt: pve02-mail-01: pve02-mail-01-crypt: pve02-mail-02: pve02-mail-02-crypt: pve02-mail-03: pve02-mail-03-crypt: pve02-mail-04: pve02-mail-04-crypt: pve03-mail-01: pve03-mail-01-crypt: pve03-mail-02: pve03-mail-02-crypt: pve03-mail-03: pve03-mail-03-crypt: pve03-mail-04: pve03-mail-04-crypt: === ACTIVE_VZDUMP === 3930634 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat "$U" >/dev/null 2>&1 then systemctl show "$U" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e "$F" then stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F" if /usr/bin/bash -n "$F" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f "$H" then echo HEALTH_FILE=$H stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H" sed -n "1p" "$H" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true === LOCAL_ARCHIVES === ARCHIVE_COUNT=10 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst SIZE=284443608 MTIME=2026-07-19T03:47:41.8525503200 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst SIZE=286538526 MTIME=2026-07-20T03:30:18.4386603270 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst SIZE=288349157 MTIME=2026-07-21T03:30:21.2480085730 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_22-03_30_01.tar.zst SIZE=290059742 MTIME=2026-07-22T03:30:22.8748248510 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst SIZE=220049041 MTIME=2026-07-19T03:47:58.9256407960 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst SIZE=220065825 MTIME=2026-07-20T03:30:30.8287264090 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst SIZE=220080970 MTIME=2026-07-21T03:30:33.3910787180 ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_22-03_30_23.tar.zst SIZE=220095608 MTIME=2026-07-22T03:30:40.6809207660 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst SIZE=868048369 MTIME=2026-06-30T03:31:08.4903256780 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst SIZE=997503422 MTIME=2026-07-01T03:31:11.1700059310 === TARGET_UNIT_LOGS === -- No entries -- ========== NODE=pve03 ========== SSH_STATUS=OK HOST=pve03 UTC_NOW=2026-07-22T14:53:55Z === FAILED_UNITS === FAILED_UNIT_COUNT=1 homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image === TARGET_UNITS === UNIT_MISSING=homelab-backup-matrix-refresh.service UNIT_MISSING=homelab-backup-matrix-refresh.timer UNIT_MISSING=homelab-vm160-remote-backup.service UNIT_MISSING=homelab-vm160-remote-backup.timer UNIT_MISSING=homelab-cluster-admin-pull.service UNIT_MISSING=homelab-cluster-admin-pull.timer UNIT_MISSING=homelab-vm-local-dumps-2cloud.service UNIT_MISSING=homelab-vm-local-dumps-2cloud.timer UNIT_MISSING=homelab-mail-cloud-critical-upload.service UNIT_MISSING=homelab-mail-cloud-critical-upload.timer UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer UNIT_MISSING=homelab-mail-cloud-restore-drill.service UNIT_MISSING=homelab-mail-cloud-restore-drill.timer UNIT_MISSING=homelab-verified-backup-health.service UNIT_MISSING=homelab-verified-backup-health.timer UNIT_MISSING=homelab-backup-sla-health.service UNIT_MISSING=homelab-backup-sla-health.timer === RELEVANT_TIMERS === === TARGET_SCRIPT_STATE === FILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh FILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup FILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull FILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill === CURRENT_BACKUP_HEALTH === HEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt HEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt === RCLONE_REMOTE_NAMES === pve01-mail-01: pve01-mail-01-crypt: pve01-mail-02: pve01-mail-02-crypt: pve01-mail-03: pve01-mail-03-crypt: pve01-mail-04: pve01-mail-04-crypt: pve01-mail-05: pve01-mail-05-crypt: pve02-mail-01: pve02-mail-01-crypt: pve02-mail-02: pve02-mail-02-crypt: pve02-mail-03: pve02-mail-03-crypt: pve02-mail-04: pve02-mail-04-crypt: pve03-mail-01: pve03-mail-01-crypt: pve03-mail-02: pve03-mail-02-crypt: pve03-mail-03: pve03-mail-03-crypt: pve03-mail-04: pve03-mail-04-crypt: === ACTIVE_VZDUMP === 2313989 bash -c set -u echo HOST=$(hostname -s) echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ) echo === FAILED_UNITS === echo -n FAILED_UNIT_COUNT= systemctl --failed --no-legend --plain 2>/dev/null | wc -l systemctl --failed --no-legend --plain 2>/dev/null || true echo === TARGET_UNITS === for U in homelab-backup-matrix-refresh.service homelab-backup-matrix-refresh.timer homelab-vm160-remote-backup.service homelab-vm160-remote-backup.timer homelab-cluster-admin-pull.service homelab-cluster-admin-pull.timer homelab-vm-local-dumps-2cloud.service homelab-vm-local-dumps-2cloud.timer homelab-mail-cloud-critical-upload.service homelab-mail-cloud-critical-upload.timer homelab-mail-cloud-edge-vm-upload.service homelab-mail-cloud-edge-vm-upload.timer homelab-mail-cloud-nextcloud-vm-upload.service homelab-mail-cloud-nextcloud-vm-upload.timer homelab-mail-cloud-restore-drill.service homelab-mail-cloud-restore-drill.timer homelab-verified-backup-health.service homelab-verified-backup-health.timer homelab-backup-sla-health.service homelab-backup-sla-health.timer do if systemctl cat "$U" >/dev/null 2>&1 then systemctl show "$U" -p Id -p LoadState -p ActiveState -p SubState -p Result -p ExecMainStatus -p UnitFileState -p ActiveEnterTimestamp -p InactiveEnterTimestamp -p ExecStart --no-pager else echo UNIT_MISSING=$U fi done echo === RELEVANT_TIMERS === systemctl list-timers --all --no-pager --no-legend 2>/dev/null | grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" || true echo === TARGET_SCRIPT_STATE === for F in /usr/local/sbin/homelab-backup-matrix-refresh /usr/local/sbin/homelab-vm160-remote-backup /usr/local/sbin/homelab-cluster-admin-pull /usr/local/sbin/homelab-vm-local-dumps-2cloud /usr/local/sbin/homelab-mail-cloud-critical-upload /usr/local/sbin/homelab-mail-cloud-edge-vm-upload /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload /usr/local/sbin/homelab-mail-cloud-restore-drill do if test -e "$F" then stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F" if /usr/bin/bash -n "$F" 2>/dev/null then echo SCRIPT_SYNTAX=$F:OK else echo SCRIPT_SYNTAX=$F:FAIL fi else echo FILE_MISSING=$F fi done echo === CURRENT_BACKUP_HEALTH === for H in /var/lib/homelab-health/unified-backup-matrix.txt /var/lib/homelab-health/backup-framework.txt /var/lib/homelab-health/backup-sla.txt /var/lib/homelab-health/backup-coverage-matrix.txt /var/lib/homelab-health/vm-local-dumps-cloud.txt /var/lib/homelab-health/vm-local-dumps-retention.txt /var/lib/homelab-health/vm-backup.txt /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt /var/lib/homelab-health/edge-vm-offhost-freshness.txt /var/lib/homelab-health/mail-cloud-restore-drill.txt do if test -f "$H" then echo HEALTH_FILE=$H stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H" sed -n "1p" "$H" else echo HEALTH_FILE_MISSING=$H fi done echo === RCLONE_REMOTE_NAMES === if command -v rclone >/dev/null 2>&1 then rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL else echo RCLONE=NOT_INSTALLED fi echo === ACTIVE_VZDUMP === pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO echo === LOCAL_ARCHIVES === echo -n ARCHIVE_COUNT= find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf x 2>/dev/null | wc -c find /var/lib/vz/dump /mnt/pve -type f \( -name "vzdump-*.zst" -o -name "vzdump-*.lzo" -o -name "vzdump-*.gz" \) -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n" 2>/dev/null | sort echo === TARGET_UNIT_LOGS === journalctl -u homelab-backup-matrix-refresh.service -u homelab-vm160-remote-backup.service -u homelab-cluster-admin-pull.service -n 120 --no-pager -o short-iso 2>/dev/null || true === LOCAL_ARCHIVES === ARCHIVE_COUNT=8 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst SIZE=11013723404 MTIME=2026-07-19T03:33:53.3463878040 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst SIZE=11116472459 MTIME=2026-07-20T03:33:53.2839064370 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst SIZE=11043834894 MTIME=2026-07-21T03:33:54.3725716230 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 MTIME=2026-07-22T03:33:58.8961071480 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst SIZE=838951976 MTIME=2026-07-19T03:34:33.1034859540 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst SIZE=838817925 MTIME=2026-07-20T03:34:37.1719936670 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst SIZE=1137072350 MTIME=2026-07-21T03:34:37.6276811360 ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_22-03_33_59.vma.zst SIZE=1177407443 MTIME=2026-07-22T03:34:44.3412293530 === TARGET_UNIT_LOGS === -- No entries -- CHANGES_MADE=NO SECRET_PRINTED=NO PRIVATE_ADDRESS_PRINTED=NO RESULT=OK BACKUP_OFFHOST_CURRENT_STATE=COMPLETE OUTPUT_END CHAT_OUTPUT_END