CHAT_OUTPUT_BEGIN
COMMAND_ID=BACKUP-OFFHOST-CURRENT-STATE-212
STATUS=OK
RC=0
HOST=pve01
MODE=read-only
COMPONENT=backup-audit
REFERENCE_REGISTER_CHECK=OK
REFERENCE_SHA256=5763f2f8edc75fcf6f3951f8c95896d1cc112ceccb6cf5b64a383f540a4deb8e
ERROR_REGISTER_CHECK=OK
ERROR_REGISTER_SHA256=08b3a769b8e38eaa1acec915352fb328180d70caa7ff6cd9d1ec0da426262752
COMMAND_SHA256=5b9f985ab6daceaad18e56d2da3dde0d2dba7eefdfe7f75e83eee6bef0a0ad8a
DUPLICATE_FAILED_COMMAND_BLOCKED=false
EXECUTION_STARTED=true
CHANGES_MADE=false
SANITIZED=yes
SECRETS_INCLUDED=no
PRIVATE_ADDRESSES_INCLUDED=no
RAW_EVIDENCE_SHA256=9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2
SANITIZED_OUTPUT_SHA256=9cb937926db26fa8365e08f136b4e2db887816e2c6d2da5a13bb76acfb311ff2
OUTPUT_BEGIN
validation=PASS
PREVIOUS_RESULT_204=VERIFIED_OK
HISTORICAL_BACKUP_BASELINE_122=VERIFIED
REFERENCE_CHECK=OK
ERROR_REGISTER_CHECK=OK
AUDIT_GENERATED_AT=2026-07-22T14:53:51Z
=== CURRENT_REFERENCE_STATE ===
{"status":"AUTO_REFRESH_READY","next_action":null,"updated_at":"2026-07-22T14:22:14+00:00"}
=== CLUSTER_NODES ===
{"node":"pve01","status":"online","uptime":540902}
{"node":"pve02","status":"online","uptime":540921}
{"node":"pve03","status":"online","uptime":540774}
=== PVE_BACKUP_JOBS ===
[{"compress":"zstd","enabled":1,"id":"homelab-nightly-all","mailnotification":"failure","mode":"snapshot","next-run":1784766600,"prune-backups":{"keep-daily":"4"},"schedule":"03:30","storage":"local","type":"vzdump","vmid":"110,111,112,113,130,150,190"}]
=== STORAGE_STATUS ===
Name             Type     Status     Total (KiB)      Used (KiB) Available (KiB)        %
local             dir     active       102626232        42074800        55292168   41.00%
local-lvm     lvmthin     active       848797696       305906689       542891006   36.04%
========== NODE=pve01 ==========
SSH_STATUS=OK
HOST=pve01
UTC_NOW=2026-07-22T14:53:53Z
=== FAILED_UNITS ===
FAILED_UNIT_COUNT=3
homelab-backup-matrix-refresh.service loaded failed failed Generate unified homelab backup matrix state
homelab-cluster-admin-pull.service    loaded failed failed Pull cluster-admin observer health into pve01
homelab-vm160-remote-backup.service   loaded failed failed VM160 full-image backup streamed from pve02 to pve03
=== TARGET_UNITS ===
Result=exit-code
ExecMainStatus=1
ExecStart={ path=/usr/local/sbin/homelab-backup-matrix-refresh ; argv[]=/usr/local/sbin/homelab-backup-matrix-refresh ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Id=homelab-backup-matrix-refresh.service
LoadState=loaded
ActiveState=failed
SubState=failed
UnitFileState=static
ActiveEnterTimestamp=
InactiveEnterTimestamp=Wed 2026-07-22 10:11:15 MSK
Result=success
Id=homelab-backup-matrix-refresh.timer
LoadState=loaded
ActiveState=inactive
SubState=dead
UnitFileState=disabled
ActiveEnterTimestamp=
InactiveEnterTimestamp=
Result=exit-code
ExecMainStatus=1
ExecStart={ path=/usr/local/sbin/homelab-vm160-remote-backup ; argv[]=/usr/local/sbin/homelab-vm160-remote-backup ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Id=homelab-vm160-remote-backup.service
LoadState=loaded
ActiveState=failed
SubState=failed
UnitFileState=static
ActiveEnterTimestamp=
InactiveEnterTimestamp=Tue 2026-07-21 02:39:14 MSK
Result=success
Id=homelab-vm160-remote-backup.timer
LoadState=loaded
ActiveState=inactive
SubState=dead
UnitFileState=disabled
ActiveEnterTimestamp=
InactiveEnterTimestamp=
Result=exit-code
ExecMainStatus=1
ExecStart={ path=/usr/local/sbin/homelab-cluster-admin-pull ; argv[]=/usr/local/sbin/homelab-cluster-admin-pull ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Id=homelab-cluster-admin-pull.service
LoadState=loaded
ActiveState=failed
SubState=failed
UnitFileState=static
ActiveEnterTimestamp=
InactiveEnterTimestamp=Tue 2026-07-21 02:37:04 MSK
Result=success
Id=homelab-cluster-admin-pull.timer
LoadState=loaded
ActiveState=inactive
SubState=dead
UnitFileState=disabled
ActiveEnterTimestamp=
InactiveEnterTimestamp=
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; argv[]=/usr/local/sbin/homelab-vm-local-dumps-2cloud ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Id=homelab-vm-local-dumps-2cloud.service
LoadState=loaded
ActiveState=inactive
SubState=dead
UnitFileState=static
ActiveEnterTimestamp=
InactiveEnterTimestamp=Sun 2026-07-19 10:13:40 MSK
Result=success
Id=homelab-vm-local-dumps-2cloud.timer
LoadState=loaded
ActiveState=active
SubState=waiting
UnitFileState=enabled
ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK
InactiveEnterTimestamp=
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-critical-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-critical-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Id=homelab-mail-cloud-critical-upload.service
LoadState=loaded
ActiveState=inactive
SubState=dead
UnitFileState=static
ActiveEnterTimestamp=
InactiveEnterTimestamp=Wed 2026-07-22 06:46:47 MSK
Result=success
Id=homelab-mail-cloud-critical-upload.timer
LoadState=loaded
ActiveState=active
SubState=waiting
UnitFileState=enabled
ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK
InactiveEnterTimestamp=
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Id=homelab-mail-cloud-edge-vm-upload.service
LoadState=loaded
ActiveState=inactive
SubState=dead
UnitFileState=static
ActiveEnterTimestamp=
InactiveEnterTimestamp=Sun 2026-07-19 09:41:56 MSK
Result=success
Id=homelab-mail-cloud-edge-vm-upload.timer
LoadState=loaded
ActiveState=active
SubState=waiting
UnitFileState=enabled
ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK
InactiveEnterTimestamp=
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; argv[]=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Id=homelab-mail-cloud-nextcloud-vm-upload.service
LoadState=loaded
ActiveState=inactive
SubState=dead
UnitFileState=static
ActiveEnterTimestamp=
InactiveEnterTimestamp=Wed 2026-07-22 07:45:01 MSK
Result=success
Id=homelab-mail-cloud-nextcloud-vm-upload.timer
LoadState=loaded
ActiveState=active
SubState=waiting
UnitFileState=enabled
ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK
InactiveEnterTimestamp=
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-superseded-unit-ok ; argv[]=/usr/local/sbin/homelab-superseded-unit-ok homelab-mail-cloud-restore-drill.service superseded_by_safety_net_restore_check ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Id=homelab-mail-cloud-restore-drill.service
LoadState=loaded
ActiveState=inactive
SubState=dead
UnitFileState=static
ActiveEnterTimestamp=
InactiveEnterTimestamp=Mon 2026-07-20 07:25:14 MSK
Result=success
Id=homelab-mail-cloud-restore-drill.timer
LoadState=loaded
ActiveState=active
SubState=waiting
UnitFileState=enabled
ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK
InactiveEnterTimestamp=
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-health-evidence ; argv[]=/usr/local/sbin/homelab-backup-health-evidence ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Id=homelab-verified-backup-health.service
LoadState=loaded
ActiveState=inactive
SubState=dead
UnitFileState=static
ActiveEnterTimestamp=
InactiveEnterTimestamp=Wed 2026-07-22 06:40:29 MSK
Result=success
Id=homelab-verified-backup-health.timer
LoadState=loaded
ActiveState=active
SubState=waiting
UnitFileState=enabled
ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK
InactiveEnterTimestamp=
Result=success
ExecMainStatus=0
ExecStart={ path=/usr/local/sbin/homelab-backup-sla-health ; argv[]=/usr/local/sbin/homelab-backup-sla-health ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 }
Id=homelab-backup-sla-health.service
LoadState=loaded
ActiveState=inactive
SubState=dead
UnitFileState=static
ActiveEnterTimestamp=
InactiveEnterTimestamp=Wed 2026-07-22 08:10:42 MSK
Result=success
Id=homelab-backup-sla-health.timer
LoadState=loaded
ActiveState=active
SubState=waiting
UnitFileState=enabled
ActiveEnterTimestamp=Thu 2026-07-16 11:38:48 MSK
InactiveEnterTimestamp=
=== RELEVANT_TIMERS ===
Thu 2026-07-23 06:38:07 MSK              12h Wed 2026-07-22 06:39:05 MSK      11h ago homelab-verified-backup-health.timer                   homelab-verified-backup-health.service
Thu 2026-07-23 06:44:08 MSK              12h Wed 2026-07-22 06:44:25 MSK      11h ago homelab-mail-cloud-critical-upload.timer               homelab-mail-cloud-critical-upload.service
Thu 2026-07-23 07:58:35 MSK              14h Wed 2026-07-22 07:42:14 MSK      10h ago homelab-mail-cloud-nextcloud-vm-upload.timer           homelab-mail-cloud-nextcloud-vm-upload.service
Thu 2026-07-23 08:12:51 MSK              14h Wed 2026-07-22 08:10:41 MSK       9h ago homelab-backup-sla-health.timer                        homelab-backup-sla-health.service
Sun 2026-07-26 09:46:53 MSK           3 days Sun 2026-07-19 09:32:29 MSK   3 days ago homelab-mail-cloud-edge-vm-upload.timer                homelab-mail-cloud-edge-vm-upload.service
Sun 2026-07-26 10:23:39 MSK           3 days Sun 2026-07-19 10:06:07 MSK   3 days ago homelab-vm-local-dumps-2cloud.timer                    homelab-vm-local-dumps-2cloud.service
Mon 2026-07-27 07:34:27 MSK           4 days Mon 2026-07-20 07:25:14 MSK   2 days ago homelab-mail-cloud-restore-drill.timer                 homelab-mail-cloud-restore-drill.service
=== TARGET_SCRIPT_STATE ===
FILE=/usr/local/sbin/homelab-backup-matrix-refresh TYPE=regular file MODE=755 OWNER=root:root SIZE=8714 MTIME=2026-07-10 13:02:04.846868245 +0300
SCRIPT_SYNTAX=/usr/local/sbin/homelab-backup-matrix-refresh:FAIL
FILE=/usr/local/sbin/homelab-vm160-remote-backup TYPE=regular file MODE=755 OWNER=root:root SIZE=3140 MTIME=2026-07-18 01:38:18.170629793 +0300
SCRIPT_SYNTAX=/usr/local/sbin/homelab-vm160-remote-backup:OK
FILE=/usr/local/sbin/homelab-cluster-admin-pull TYPE=regular file MODE=755 OWNER=root:root SIZE=2501 MTIME=2026-07-13 01:15:06.431144817 +0300
SCRIPT_SYNTAX=/usr/local/sbin/homelab-cluster-admin-pull:OK
FILE=/usr/local/sbin/homelab-vm-local-dumps-2cloud TYPE=regular file MODE=700 OWNER=root:root SIZE=7272 MTIME=2026-07-06 18:49:49.740645050 +0300
SCRIPT_SYNTAX=/usr/local/sbin/homelab-vm-local-dumps-2cloud:OK
FILE=/usr/local/sbin/homelab-mail-cloud-critical-upload TYPE=regular file MODE=700 OWNER=root:root SIZE=7836 MTIME=2026-07-17 14:03:50.373696143 +0300
SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-critical-upload:OK
FILE=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=12899 MTIME=2026-07-13 21:27:22.164609073 +0300
SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload:OK
FILE=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload TYPE=regular file MODE=750 OWNER=root:root SIZE=6542 MTIME=2026-06-30 15:00:33.588620116 +0300
SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload:OK
FILE=/usr/local/sbin/homelab-mail-cloud-restore-drill TYPE=regular file MODE=700 OWNER=root:root SIZE=7995 MTIME=2026-07-10 04:14:10.054957455 +0300
SCRIPT_SYNTAX=/usr/local/sbin/homelab-mail-cloud-restore-drill:OK
=== CURRENT_BACKUP_HEALTH ===
HEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt
HEALTH_FILE=/var/lib/homelab-health/backup-framework.txt
HEALTH_MTIME=2026-07-22 10:11:15.588809944 +0300 HEALTH_SIZE=803
STATUS=OK
HEALTH_FILE=/var/lib/homelab-health/backup-sla.txt
HEALTH_MTIME=2026-07-22 17:45:04.576070554 +0300 HEALTH_SIZE=154
STATUS=WARN TS=2026-07-22T14:45:04Z TYPE=backup-sla BAD=0 WARN=2 APPS=45 EXPECTED=12 MAX_AGE_HOURS=72 REPORT=/var/lib/homelab-backup/backup-sla-latest.md
HEALTH_FILE=/var/lib/homelab-health/backup-coverage-matrix.txt
HEALTH_MTIME=2026-07-22 10:11:15.236509358 +0300 HEALTH_SIZE=808
STATUS=OK
HEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-cloud.txt
HEALTH_MTIME=2026-07-22 10:11:15.694007276 +0300 HEALTH_SIZE=800
STATUS=OK
HEALTH_FILE=/var/lib/homelab-health/vm-local-dumps-retention.txt
HEALTH_MTIME=2026-07-22 10:11:15.711074323 +0300 HEALTH_SIZE=826
STATUS=OK
HEALTH_FILE=/var/lib/homelab-health/vm-backup.txt
HEALTH_MTIME=2026-07-22 07:35:14.956502473 +0300 HEALTH_SIZE=170
STATUS=OK TS=2026-07-22T04:35:14Z TYPE=vm-backup BAD=0 auto-vmct=OK edge-vm-backup=OK edge-vm-offhost=OK edge-vm-restore=OK auto-timer=active edge-freshness-timer=active
HEALTH_FILE=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt
HEALTH_MTIME=2026-07-09 08:07:47.213452046 +0300 HEALTH_SIZE=217
STATUS=OK TS=2026-07-09T05:07:47Z TYPE=cluster-admin-vm-mail-cloud-backup BAD=0 VMID=180 NODE=pve02 MODE=MAIL_CLOUD_2COPY_VERIFY_OK LOCAL_STATUS=OK REPORT=/var/lib/homelab-cluster-admin-vm-mail-cloud-backup/latest.md
HEALTH_FILE=/var/lib/homelab-health/edge-vm-offhost-freshness.txt
HEALTH_MTIME=2026-07-22 04:46:23.189237991 +0300 HEALTH_SIZE=409
STATUS=OK TS=2026-07-22T01:39:30Z TYPE=edge-vm-offhost-freshness BAD=0 VMID=130 SOURCE_NODE=pve03 DEST_NODE=pve02 SRC=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst DEST=/mnt/staging/offhost/edge-vm-vzdump-from-pve03/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 SHA256=a8e5179614e02e6933617adb30cb95a35a1b5d17b30c9f00a2626cfa06b14710 COPY_STATUS=copied ZSTD_RC=0 VMA_RC=0 KEEP=4
HEALTH_FILE=/var/lib/homelab-health/mail-cloud-restore-drill.txt
HEALTH_MTIME=2026-07-22 10:11:15.491775958 +0300 HEALTH_SIZE=830
STATUS=STALE
=== RCLONE_REMOTE_NAMES ===
pve01-mail-01:
pve01-mail-01-crypt:
pve01-mail-02:
pve01-mail-02-crypt:
pve01-mail-03:
pve01-mail-03-crypt:
pve01-mail-04:
pve01-mail-04-crypt:
pve01-mail-05:
pve01-mail-05-crypt:
pve02-mail-01:
pve02-mail-01-crypt:
pve02-mail-02:
pve02-mail-02-crypt:
pve02-mail-03:
pve02-mail-03-crypt:
pve02-mail-04:
pve02-mail-04-crypt:
pve03-mail-01:
pve03-mail-01-crypt:
pve03-mail-02:
pve03-mail-02-crypt:
pve03-mail-03:
pve03-mail-03-crypt:
pve03-mail-04:
pve03-mail-04-crypt:
=== ACTIVE_VZDUMP ===
1599768 bash /usr/local/sbin/homelab-chat-run BACKUP-OFFHOST-CURRENT-STATE-212 read-only backup-audit -- /usr/bin/bash -c  set -eu  test "$(hostname -s)" = "pve01" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null  /usr/local/sbin/homelab-reference validate  W="$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)" trap "rm -rf -- \"$W\"" EXIT  BASE="https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history"  curl -fsSL \     "$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json" \     -o "$W/204.json"  jq -e "     .command_id == \"HOMELAB-TAILS-FINAL-CLOSEOUT-204\" and     .status == \"OK\" and     ((.rc | tonumber) == 0) and     .reference_register_checked and     .error_register_checked and     ((.output // \"\") | contains(\"ALL_TAILS_CLOSED=YES\")) and     ((.output // \"\") | contains(\"TAILS_FINAL_CLOSEOUT=COMPLETE\")) " "$W/204.json" >/dev/null  curl -fsSL \     "$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json" \     -o "$W/122.json"  jq -e "     .command_id == \"HOMELAB-INDEPENDENT-FINAL-STATE-122\" and     .status == \"OK\" and     ((.rc | tonumber) == 0) and     ((.output // \"\") | contains(\"HOMELAB_FINAL_STATE=BLOCKED\")) and     ((.output // \"\") | contains(\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\")) " "$W/122.json" >/dev/null  echo "PREVIOUS_RESULT_204=VERIFIED_OK" echo "HISTORICAL_BACKUP_BASELINE_122=VERIFIED" echo "REFERENCE_CHECK=OK" echo "ERROR_REGISTER_CHECK=OK" echo "AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)"  echo "=== CURRENT_REFERENCE_STATE ==="  STATE="/etc/homelab/reference/state/current.json"  if test -s "$STATE" && jq -e "." "$STATE" >/dev/null; then     jq -c "{status,next_action,updated_at}" "$STATE" else     echo "REFERENCE_STATE=INVALID_OR_MISSING" fi  echo "=== CLUSTER_NODES ===" pvesh get /nodes --output-format json | jq -c "sort_by(.node)[] | {node,status,uptime}"  echo "=== PVE_BACKUP_JOBS ===" pvesh get /cluster/backup --output-format json | jq -c "sort_by(.id)"  echo "=== STORAGE_STATUS ===" pvesm status || true  SSH_OPTS=(     -o BatchMode=yes     -o ConnectTimeout=10     -o ConnectionAttempts=1     -o StrictHostKeyChecking=yes )  for NODE in pve01 pve02 pve03 do     echo "========== NODE=$NODE =========="      if ssh "${SSH_OPTS[@]}" root@"$NODE" true     then         echo "SSH_STATUS=OK"          ssh "${SSH_OPTS[@]}" root@"$NODE" "             set -u              echo HOST=\$(hostname -s)             echo UTC_NOW=\$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in \                 homelab-backup-matrix-refresh.service \                 homelab-backup-matrix-refresh.timer \                 homelab-vm160-remote-backup.service \                 homelab-vm160-remote-backup.timer \                 homelab-cluster-admin-pull.service \                 homelab-cluster-admin-pull.timer \                 homelab-vm-local-dumps-2cloud.service \                 homelab-vm-local-dumps-2cloud.timer \                 homelab-mail-cloud-critical-upload.service \                 homelab-mail-cloud-critical-upload.timer \                 homelab-mail-cloud-edge-vm-upload.service \                 homelab-mail-cloud-edge-vm-upload.timer \                 homelab-mail-cloud-nextcloud-vm-upload.service \                 homelab-mail-cloud-nextcloud-vm-upload.timer \                 homelab-mail-cloud-restore-drill.service \                 homelab-mail-cloud-restore-drill.timer \                 homelab-verified-backup-health.service \                 homelab-verified-backup-health.timer \                 homelab-backup-sla-health.service \                 homelab-backup-sla-health.timer             do                 if systemctl cat \"\$U\" >/dev/null 2>&1                 then                     systemctl show \"\$U\" \                         -p Id \                         -p LoadState \                         -p ActiveState \                         -p SubState \                         -p Result \                         -p ExecMainStatus \                         -p UnitFileState \                         -p ActiveEnterTimestamp \                         -p InactiveEnterTimestamp \                         -p ExecStart \                         --no-pager                 else                     echo UNIT_MISSING=\$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in \                 /usr/local/sbin/homelab-backup-matrix-refresh \                 /usr/local/sbin/homelab-vm160-remote-backup \                 /usr/local/sbin/homelab-cluster-admin-pull \                 /usr/local/sbin/homelab-vm-local-dumps-2cloud \                 /usr/local/sbin/homelab-mail-cloud-critical-upload \                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e \"\$F\"                 then                     stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"\$F\"                      if /usr/bin/bash -n \"\$F\" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=\$F:OK                     else                         echo SCRIPT_SYNTAX=\$F:FAIL                     fi                 else                     echo FILE_MISSING=\$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in \                 /var/lib/homelab-health/unified-backup-matrix.txt \                 /var/lib/homelab-health/backup-framework.txt \                 /var/lib/homelab-health/backup-sla.txt \                 /var/lib/homelab-health/backup-coverage-matrix.txt \                 /var/lib/homelab-health/vm-local-dumps-cloud.txt \                 /var/lib/homelab-health/vm-local-dumps-retention.txt \                 /var/lib/homelab-health/vm-backup.txt \                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt \                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f \"\$H\"                 then                     echo HEALTH_FILE=\$H                     stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"\$H\"                     sed -n \"1p\" \"\$H\"                 else                     echo HEALTH_FILE_MISSING=\$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve \                 -type f \                 \( -name \"vzdump-*.zst\" \                    -o -name \"vzdump-*.lzo\" \                    -o -name \"vzdump-*.gz\" \) \                 -printf x \                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve \                 -type f \                 \( -name \"vzdump-*.zst\" \                    -o -name \"vzdump-*.lzo\" \                    -o -name \"vzdump-*.gz\" \) \                 -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n\" \                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl \                 -u homelab-backup-matrix-refresh.service \                 -u homelab-vm160-remote-backup.service \                 -u homelab-cluster-admin-pull.service \                 -n 120 \                 --no-pager \                 -o short-iso \                 2>/dev/null ||                 true         "     else         echo "SSH_STATUS=FAIL"     fi done  echo "CHANGES_MADE=NO" echo "SECRET_PRINTED=NO" echo "PRIVATE_ADDRESS_PRINTED=NO" echo "RESULT=OK" echo "BACKUP_OFFHOST_CURRENT_STATE=COMPLETE" 
1599834 timeout --foreground --preserve-status 300 /usr/bin/bash -c  set -eu  test "$(hostname -s)" = "pve01" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null  /usr/local/sbin/homelab-reference validate  W="$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)" trap "rm -rf -- \"$W\"" EXIT  BASE="https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history"  curl -fsSL \     "$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json" \     -o "$W/204.json"  jq -e "     .command_id == \"HOMELAB-TAILS-FINAL-CLOSEOUT-204\" and     .status == \"OK\" and     ((.rc | tonumber) == 0) and     .reference_register_checked and     .error_register_checked and     ((.output // \"\") | contains(\"ALL_TAILS_CLOSED=YES\")) and     ((.output // \"\") | contains(\"TAILS_FINAL_CLOSEOUT=COMPLETE\")) " "$W/204.json" >/dev/null  curl -fsSL \     "$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json" \     -o "$W/122.json"  jq -e "     .command_id == \"HOMELAB-INDEPENDENT-FINAL-STATE-122\" and     .status == \"OK\" and     ((.rc | tonumber) == 0) and     ((.output // \"\") | contains(\"HOMELAB_FINAL_STATE=BLOCKED\")) and     ((.output // \"\") | contains(\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\")) " "$W/122.json" >/dev/null  echo "PREVIOUS_RESULT_204=VERIFIED_OK" echo "HISTORICAL_BACKUP_BASELINE_122=VERIFIED" echo "REFERENCE_CHECK=OK" echo "ERROR_REGISTER_CHECK=OK" echo "AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)"  echo "=== CURRENT_REFERENCE_STATE ==="  STATE="/etc/homelab/reference/state/current.json"  if test -s "$STATE" && jq -e "." "$STATE" >/dev/null; then     jq -c "{status,next_action,updated_at}" "$STATE" else     echo "REFERENCE_STATE=INVALID_OR_MISSING" fi  echo "=== CLUSTER_NODES ===" pvesh get /nodes --output-format json | jq -c "sort_by(.node)[] | {node,status,uptime}"  echo "=== PVE_BACKUP_JOBS ===" pvesh get /cluster/backup --output-format json | jq -c "sort_by(.id)"  echo "=== STORAGE_STATUS ===" pvesm status || true  SSH_OPTS=(     -o BatchMode=yes     -o ConnectTimeout=10     -o ConnectionAttempts=1     -o StrictHostKeyChecking=yes )  for NODE in pve01 pve02 pve03 do     echo "========== NODE=$NODE =========="      if ssh "${SSH_OPTS[@]}" root@"$NODE" true     then         echo "SSH_STATUS=OK"          ssh "${SSH_OPTS[@]}" root@"$NODE" "             set -u              echo HOST=\$(hostname -s)             echo UTC_NOW=\$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in \                 homelab-backup-matrix-refresh.service \                 homelab-backup-matrix-refresh.timer \                 homelab-vm160-remote-backup.service \                 homelab-vm160-remote-backup.timer \                 homelab-cluster-admin-pull.service \                 homelab-cluster-admin-pull.timer \                 homelab-vm-local-dumps-2cloud.service \                 homelab-vm-local-dumps-2cloud.timer \                 homelab-mail-cloud-critical-upload.service \                 homelab-mail-cloud-critical-upload.timer \                 homelab-mail-cloud-edge-vm-upload.service \                 homelab-mail-cloud-edge-vm-upload.timer \                 homelab-mail-cloud-nextcloud-vm-upload.service \                 homelab-mail-cloud-nextcloud-vm-upload.timer \                 homelab-mail-cloud-restore-drill.service \                 homelab-mail-cloud-restore-drill.timer \                 homelab-verified-backup-health.service \                 homelab-verified-backup-health.timer \                 homelab-backup-sla-health.service \                 homelab-backup-sla-health.timer             do                 if systemctl cat \"\$U\" >/dev/null 2>&1                 then                     systemctl show \"\$U\" \                         -p Id \                         -p LoadState \                         -p ActiveState \                         -p SubState \                         -p Result \                         -p ExecMainStatus \                         -p UnitFileState \                         -p ActiveEnterTimestamp \                         -p InactiveEnterTimestamp \                         -p ExecStart \                         --no-pager                 else                     echo UNIT_MISSING=\$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in \                 /usr/local/sbin/homelab-backup-matrix-refresh \                 /usr/local/sbin/homelab-vm160-remote-backup \                 /usr/local/sbin/homelab-cluster-admin-pull \                 /usr/local/sbin/homelab-vm-local-dumps-2cloud \                 /usr/local/sbin/homelab-mail-cloud-critical-upload \                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e \"\$F\"                 then                     stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"\$F\"                      if /usr/bin/bash -n \"\$F\" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=\$F:OK                     else                         echo SCRIPT_SYNTAX=\$F:FAIL                     fi                 else                     echo FILE_MISSING=\$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in \                 /var/lib/homelab-health/unified-backup-matrix.txt \                 /var/lib/homelab-health/backup-framework.txt \                 /var/lib/homelab-health/backup-sla.txt \                 /var/lib/homelab-health/backup-coverage-matrix.txt \                 /var/lib/homelab-health/vm-local-dumps-cloud.txt \                 /var/lib/homelab-health/vm-local-dumps-retention.txt \                 /var/lib/homelab-health/vm-backup.txt \                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt \                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f \"\$H\"                 then                     echo HEALTH_FILE=\$H                     stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"\$H\"                     sed -n \"1p\" \"\$H\"                 else                     echo HEALTH_FILE_MISSING=\$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve \                 -type f \                 \( -name \"vzdump-*.zst\" \                    -o -name \"vzdump-*.lzo\" \                    -o -name \"vzdump-*.gz\" \) \                 -printf x \                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve \                 -type f \                 \( -name \"vzdump-*.zst\" \                    -o -name \"vzdump-*.lzo\" \                    -o -name \"vzdump-*.gz\" \) \                 -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n\" \                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl \                 -u homelab-backup-matrix-refresh.service \                 -u homelab-vm160-remote-backup.service \                 -u homelab-cluster-admin-pull.service \                 -n 120 \                 --no-pager \                 -o short-iso \                 2>/dev/null ||                 true         "     else         echo "SSH_STATUS=FAIL"     fi done  echo "CHANGES_MADE=NO" echo "SECRET_PRINTED=NO" echo "PRIVATE_ADDRESS_PRINTED=NO" echo "RESULT=OK" echo "BACKUP_OFFHOST_CURRENT_STATE=COMPLETE" 
1599836 /usr/bin/bash -c  set -eu  test "$(hostname -s)" = "pve01" test -s /etc/pve/31_HOMELAB_REFERENCE.md test -s /etc/pve/HOMELAB_ASSISTANT_ERROR_REGISTER.md command -v curl >/dev/null command -v jq >/dev/null command -v ssh >/dev/null  /usr/local/sbin/homelab-reference validate  W="$(mktemp -d /var/tmp/backup-current-state-212.XXXXXX)" trap "rm -rf -- \"$W\"" EXIT  BASE="https://git.gram1.ru/homelab-admin/homelab-public-context/raw/branch/assistant-runtime/runtime/history"  curl -fsSL \     "$BASE/HOMELAB-TAILS-FINAL-CLOSEOUT-204.json" \     -o "$W/204.json"  jq -e "     .command_id == \"HOMELAB-TAILS-FINAL-CLOSEOUT-204\" and     .status == \"OK\" and     ((.rc | tonumber) == 0) and     .reference_register_checked and     .error_register_checked and     ((.output // \"\") | contains(\"ALL_TAILS_CLOSED=YES\")) and     ((.output // \"\") | contains(\"TAILS_FINAL_CLOSEOUT=COMPLETE\")) " "$W/204.json" >/dev/null  curl -fsSL \     "$BASE/HOMELAB-INDEPENDENT-FINAL-STATE-122.json" \     -o "$W/122.json"  jq -e "     .command_id == \"HOMELAB-INDEPENDENT-FINAL-STATE-122\" and     .status == \"OK\" and     ((.rc | tonumber) == 0) and     ((.output // \"\") | contains(\"HOMELAB_FINAL_STATE=BLOCKED\")) and     ((.output // \"\") | contains(\"NEXT_ACTION=RECONCILE_DUAL_MAIL_CLOUD_BACKUPS\")) " "$W/122.json" >/dev/null  echo "PREVIOUS_RESULT_204=VERIFIED_OK" echo "HISTORICAL_BACKUP_BASELINE_122=VERIFIED" echo "REFERENCE_CHECK=OK" echo "ERROR_REGISTER_CHECK=OK" echo "AUDIT_GENERATED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)"  echo "=== CURRENT_REFERENCE_STATE ==="  STATE="/etc/homelab/reference/state/current.json"  if test -s "$STATE" && jq -e "." "$STATE" >/dev/null; then     jq -c "{status,next_action,updated_at}" "$STATE" else     echo "REFERENCE_STATE=INVALID_OR_MISSING" fi  echo "=== CLUSTER_NODES ===" pvesh get /nodes --output-format json | jq -c "sort_by(.node)[] | {node,status,uptime}"  echo "=== PVE_BACKUP_JOBS ===" pvesh get /cluster/backup --output-format json | jq -c "sort_by(.id)"  echo "=== STORAGE_STATUS ===" pvesm status || true  SSH_OPTS=(     -o BatchMode=yes     -o ConnectTimeout=10     -o ConnectionAttempts=1     -o StrictHostKeyChecking=yes )  for NODE in pve01 pve02 pve03 do     echo "========== NODE=$NODE =========="      if ssh "${SSH_OPTS[@]}" root@"$NODE" true     then         echo "SSH_STATUS=OK"          ssh "${SSH_OPTS[@]}" root@"$NODE" "             set -u              echo HOST=\$(hostname -s)             echo UTC_NOW=\$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in \                 homelab-backup-matrix-refresh.service \                 homelab-backup-matrix-refresh.timer \                 homelab-vm160-remote-backup.service \                 homelab-vm160-remote-backup.timer \                 homelab-cluster-admin-pull.service \                 homelab-cluster-admin-pull.timer \                 homelab-vm-local-dumps-2cloud.service \                 homelab-vm-local-dumps-2cloud.timer \                 homelab-mail-cloud-critical-upload.service \                 homelab-mail-cloud-critical-upload.timer \                 homelab-mail-cloud-edge-vm-upload.service \                 homelab-mail-cloud-edge-vm-upload.timer \                 homelab-mail-cloud-nextcloud-vm-upload.service \                 homelab-mail-cloud-nextcloud-vm-upload.timer \                 homelab-mail-cloud-restore-drill.service \                 homelab-mail-cloud-restore-drill.timer \                 homelab-verified-backup-health.service \                 homelab-verified-backup-health.timer \                 homelab-backup-sla-health.service \                 homelab-backup-sla-health.timer             do                 if systemctl cat \"\$U\" >/dev/null 2>&1                 then                     systemctl show \"\$U\" \                         -p Id \                         -p LoadState \                         -p ActiveState \                         -p SubState \                         -p Result \                         -p ExecMainStatus \                         -p UnitFileState \                         -p ActiveEnterTimestamp \                         -p InactiveEnterTimestamp \                         -p ExecStart \                         --no-pager                 else                     echo UNIT_MISSING=\$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E \"homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)\" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in \                 /usr/local/sbin/homelab-backup-matrix-refresh \                 /usr/local/sbin/homelab-vm160-remote-backup \                 /usr/local/sbin/homelab-cluster-admin-pull \                 /usr/local/sbin/homelab-vm-local-dumps-2cloud \                 /usr/local/sbin/homelab-mail-cloud-critical-upload \                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload \                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload \                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e \"\$F\"                 then                     stat -c \"FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y\" \"\$F\"                      if /usr/bin/bash -n \"\$F\" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=\$F:OK                     else                         echo SCRIPT_SYNTAX=\$F:FAIL                     fi                 else                     echo FILE_MISSING=\$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in \                 /var/lib/homelab-health/unified-backup-matrix.txt \                 /var/lib/homelab-health/backup-framework.txt \                 /var/lib/homelab-health/backup-sla.txt \                 /var/lib/homelab-health/backup-coverage-matrix.txt \                 /var/lib/homelab-health/vm-local-dumps-cloud.txt \                 /var/lib/homelab-health/vm-local-dumps-retention.txt \                 /var/lib/homelab-health/vm-backup.txt \                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt \                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt \                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f \"\$H\"                 then                     echo HEALTH_FILE=\$H                     stat -c \"HEALTH_MTIME=%y HEALTH_SIZE=%s\" \"\$H\"                     sed -n \"1p\" \"\$H\"                 else                     echo HEALTH_FILE_MISSING=\$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af \"[v]zdump\" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve \                 -type f \                 \( -name \"vzdump-*.zst\" \                    -o -name \"vzdump-*.lzo\" \                    -o -name \"vzdump-*.gz\" \) \                 -printf x \                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve \                 -type f \                 \( -name \"vzdump-*.zst\" \                    -o -name \"vzdump-*.lzo\" \                    -o -name \"vzdump-*.gz\" \) \                 -printf \"ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n\" \                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl \                 -u homelab-backup-matrix-refresh.service \                 -u homelab-vm160-remote-backup.service \                 -u homelab-cluster-admin-pull.service \                 -n 120 \                 --no-pager \                 -o short-iso \                 2>/dev/null ||                 true         "     else         echo "SSH_STATUS=FAIL"     fi done  echo "CHANGES_MADE=NO" echo "SECRET_PRINTED=NO" echo "PRIVATE_ADDRESS_PRINTED=NO" echo "RESULT=OK" echo "BACKUP_OFFHOST_CURRENT_STATE=COMPLETE" 
1603200 ssh -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o StrictHostKeyChecking=yes root@pve01              set -u              echo HOST=$(hostname -s)             echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in                 homelab-backup-matrix-refresh.service                 homelab-backup-matrix-refresh.timer                 homelab-vm160-remote-backup.service                 homelab-vm160-remote-backup.timer                 homelab-cluster-admin-pull.service                 homelab-cluster-admin-pull.timer                 homelab-vm-local-dumps-2cloud.service                 homelab-vm-local-dumps-2cloud.timer                 homelab-mail-cloud-critical-upload.service                 homelab-mail-cloud-critical-upload.timer                 homelab-mail-cloud-edge-vm-upload.service                 homelab-mail-cloud-edge-vm-upload.timer                 homelab-mail-cloud-nextcloud-vm-upload.service                 homelab-mail-cloud-nextcloud-vm-upload.timer                 homelab-mail-cloud-restore-drill.service                 homelab-mail-cloud-restore-drill.timer                 homelab-verified-backup-health.service                 homelab-verified-backup-health.timer                 homelab-backup-sla-health.service                 homelab-backup-sla-health.timer             do                 if systemctl cat "$U" >/dev/null 2>&1                 then                     systemctl show "$U"                         -p Id                         -p LoadState                         -p ActiveState                         -p SubState                         -p Result                         -p ExecMainStatus                         -p UnitFileState                         -p ActiveEnterTimestamp                         -p InactiveEnterTimestamp                         -p ExecStart                         --no-pager                 else                     echo UNIT_MISSING=$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in                 /usr/local/sbin/homelab-backup-matrix-refresh                 /usr/local/sbin/homelab-vm160-remote-backup                 /usr/local/sbin/homelab-cluster-admin-pull                 /usr/local/sbin/homelab-vm-local-dumps-2cloud                 /usr/local/sbin/homelab-mail-cloud-critical-upload                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e "$F"                 then                     stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F"                      if /usr/bin/bash -n "$F" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=$F:OK                     else                         echo SCRIPT_SYNTAX=$F:FAIL                     fi                 else                     echo FILE_MISSING=$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in                 /var/lib/homelab-health/unified-backup-matrix.txt                 /var/lib/homelab-health/backup-framework.txt                 /var/lib/homelab-health/backup-sla.txt                 /var/lib/homelab-health/backup-coverage-matrix.txt                 /var/lib/homelab-health/vm-local-dumps-cloud.txt                 /var/lib/homelab-health/vm-local-dumps-retention.txt                 /var/lib/homelab-health/vm-backup.txt                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f "$H"                 then                     echo HEALTH_FILE=$H                     stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H"                     sed -n "1p" "$H"                 else                     echo HEALTH_FILE_MISSING=$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve                 -type f                 \( -name "vzdump-*.zst"                    -o -name "vzdump-*.lzo"                    -o -name "vzdump-*.gz" \)                 -printf x                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve                 -type f                 \( -name "vzdump-*.zst"                    -o -name "vzdump-*.lzo"                    -o -name "vzdump-*.gz" \)                 -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n"                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl                 -u homelab-backup-matrix-refresh.service                 -u homelab-vm160-remote-backup.service                 -u homelab-cluster-admin-pull.service                 -n 120                 --no-pager                 -o short-iso                 2>/dev/null ||                 true         
1603210 bash -c              set -u              echo HOST=$(hostname -s)             echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in                 homelab-backup-matrix-refresh.service                 homelab-backup-matrix-refresh.timer                 homelab-vm160-remote-backup.service                 homelab-vm160-remote-backup.timer                 homelab-cluster-admin-pull.service                 homelab-cluster-admin-pull.timer                 homelab-vm-local-dumps-2cloud.service                 homelab-vm-local-dumps-2cloud.timer                 homelab-mail-cloud-critical-upload.service                 homelab-mail-cloud-critical-upload.timer                 homelab-mail-cloud-edge-vm-upload.service                 homelab-mail-cloud-edge-vm-upload.timer                 homelab-mail-cloud-nextcloud-vm-upload.service                 homelab-mail-cloud-nextcloud-vm-upload.timer                 homelab-mail-cloud-restore-drill.service                 homelab-mail-cloud-restore-drill.timer                 homelab-verified-backup-health.service                 homelab-verified-backup-health.timer                 homelab-backup-sla-health.service                 homelab-backup-sla-health.timer             do                 if systemctl cat "$U" >/dev/null 2>&1                 then                     systemctl show "$U"                         -p Id                         -p LoadState                         -p ActiveState                         -p SubState                         -p Result                         -p ExecMainStatus                         -p UnitFileState                         -p ActiveEnterTimestamp                         -p InactiveEnterTimestamp                         -p ExecStart                         --no-pager                 else                     echo UNIT_MISSING=$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in                 /usr/local/sbin/homelab-backup-matrix-refresh                 /usr/local/sbin/homelab-vm160-remote-backup                 /usr/local/sbin/homelab-cluster-admin-pull                 /usr/local/sbin/homelab-vm-local-dumps-2cloud                 /usr/local/sbin/homelab-mail-cloud-critical-upload                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e "$F"                 then                     stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F"                      if /usr/bin/bash -n "$F" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=$F:OK                     else                         echo SCRIPT_SYNTAX=$F:FAIL                     fi                 else                     echo FILE_MISSING=$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in                 /var/lib/homelab-health/unified-backup-matrix.txt                 /var/lib/homelab-health/backup-framework.txt                 /var/lib/homelab-health/backup-sla.txt                 /var/lib/homelab-health/backup-coverage-matrix.txt                 /var/lib/homelab-health/vm-local-dumps-cloud.txt                 /var/lib/homelab-health/vm-local-dumps-retention.txt                 /var/lib/homelab-health/vm-backup.txt                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f "$H"                 then                     echo HEALTH_FILE=$H                     stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H"                     sed -n "1p" "$H"                 else                     echo HEALTH_FILE_MISSING=$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve                 -type f                 \( -name "vzdump-*.zst"                    -o -name "vzdump-*.lzo"                    -o -name "vzdump-*.gz" \)                 -printf x                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve                 -type f                 \( -name "vzdump-*.zst"                    -o -name "vzdump-*.lzo"                    -o -name "vzdump-*.gz" \)                 -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n"                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl                 -u homelab-backup-matrix-refresh.service                 -u homelab-vm160-remote-backup.service                 -u homelab-cluster-admin-pull.service                 -n 120                 --no-pager                 -o short-iso                 2>/dev/null ||                 true         
=== LOCAL_ARCHIVES ===
ARCHIVE_COUNT=12
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_19-03_45_38.tar.zst SIZE=323668231 MTIME=2026-07-19T03:45:46.2209130400
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_20-03_30_00.tar.zst SIZE=326601118 MTIME=2026-07-20T03:30:08.2096782720
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_21-03_30_02.tar.zst SIZE=329541843 MTIME=2026-07-21T03:30:09.7688696330
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-110-2026_07_22-03_30_01.tar.zst SIZE=332951369 MTIME=2026-07-22T03:30:08.7686076500
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_19-03_45_47.tar.zst SIZE=222745965 MTIME=2026-07-19T03:45:57.2700790730
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_20-03_30_08.tar.zst SIZE=222761273 MTIME=2026-07-20T03:30:13.7937629440
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_21-03_30_10.tar.zst SIZE=222777732 MTIME=2026-07-21T03:30:15.9059638290
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-112-2026_07_22-03_30_09.tar.zst SIZE=222794054 MTIME=2026-07-22T03:30:14.2846933120
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_19-03_30_16.vma.zst SIZE=7021297699 MTIME=2026-07-19T03:31:06.8755351020
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_20-03_30_14.vma.zst SIZE=7016275481 MTIME=2026-07-20T03:31:03.2165123470
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_21-03_30_16.vma.zst SIZE=7021952392 MTIME=2026-07-21T03:31:06.5717414950
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-150-2026_07_22-03_30_14.vma.zst SIZE=7052176114 MTIME=2026-07-22T03:31:02.8954482120
=== TARGET_UNIT_LOGS ===
2026-07-22T05:48:02+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_STATUS=FAIL
2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_TOTAL=41
2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_OK=30
2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_BAD=6
2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_UNMAPPED=0
2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JOBS_DISABLED=5
2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
2026-07-22T05:48:03+03:00 pve01 homelab-backup-matrix-refresh[3328302]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
2026-07-22T05:48:03+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
2026-07-22T05:48:03+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
2026-07-22T06:11:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_STATUS=FAIL
2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_TOTAL=41
2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_OK=30
2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_BAD=6
2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_UNMAPPED=0
2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JOBS_DISABLED=5
2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
2026-07-22T06:11:13+03:00 pve01 homelab-backup-matrix-refresh[3416905]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
2026-07-22T06:11:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
2026-07-22T06:11:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
2026-07-22T06:40:29+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_STATUS=FAIL
2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_TOTAL=41
2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_OK=30
2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_BAD=6
2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_UNMAPPED=0
2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JOBS_DISABLED=5
2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
2026-07-22T06:40:30+03:00 pve01 homelab-backup-matrix-refresh[3508155]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
2026-07-22T06:40:30+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
2026-07-22T06:40:30+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
2026-07-22T07:02:40+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_STATUS=FAIL
2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_TOTAL=41
2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_OK=30
2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_BAD=6
2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_UNMAPPED=0
2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JOBS_DISABLED=5
2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
2026-07-22T07:02:41+03:00 pve01 homelab-backup-matrix-refresh[3582804]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
2026-07-22T07:02:41+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
2026-07-22T07:02:41+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
2026-07-22T07:35:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_STATUS=FAIL
2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_TOTAL=41
2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_OK=30
2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_BAD=6
2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_UNMAPPED=0
2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JOBS_DISABLED=5
2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
2026-07-22T07:35:15+03:00 pve01 homelab-backup-matrix-refresh[3686599]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
2026-07-22T07:35:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
2026-07-22T07:35:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
2026-07-22T08:10:42+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_STATUS=FAIL
2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_TOTAL=41
2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_OK=30
2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_BAD=6
2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_UNMAPPED=0
2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JOBS_DISABLED=5
2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
2026-07-22T08:10:43+03:00 pve01 homelab-backup-matrix-refresh[3806058]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
2026-07-22T08:10:43+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
2026-07-22T08:10:43+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
2026-07-22T08:37:12+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_STATUS=FAIL
2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_TOTAL=41
2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_OK=30
2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_BAD=6
2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_UNMAPPED=0
2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JOBS_DISABLED=5
2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
2026-07-22T08:37:13+03:00 pve01 homelab-backup-matrix-refresh[3893084]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
2026-07-22T08:37:13+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
2026-07-22T08:37:13+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
2026-07-22T08:40:15+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_STATUS=FAIL
2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_TOTAL=41
2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_OK=30
2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_BAD=6
2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_UNMAPPED=0
2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JOBS_DISABLED=5
2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
2026-07-22T08:40:16+03:00 pve01 homelab-backup-matrix-refresh[3894794]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
2026-07-22T08:40:16+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
2026-07-22T08:40:16+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
2026-07-22T08:53:36+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_STATUS=FAIL
2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_TOTAL=41
2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_OK=30
2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_BAD=6
2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_UNMAPPED=0
2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JOBS_DISABLED=5
2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
2026-07-22T08:53:37+03:00 pve01 homelab-backup-matrix-refresh[3943967]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
2026-07-22T08:53:37+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
2026-07-22T08:53:37+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
2026-07-22T10:11:14+03:00 pve01 systemd[1]: Starting homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state...
2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_STATUS=FAIL
2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_TOTAL=41
2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_OK=30
2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_BAD=6
2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_UNMAPPED=0
2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JOBS_DISABLED=5
2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_STATE_JSON=/var/lib/homelab-backup-matrix/state.json
2026-07-22T10:11:15+03:00 pve01 homelab-backup-matrix-refresh[4192756]: MATRIX_RENDERED_DIR=/var/lib/homelab-backup-matrix/rendered
2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Main process exited, code=exited, status=1/FAILURE
2026-07-22T10:11:15+03:00 pve01 systemd[1]: homelab-backup-matrix-refresh.service: Failed with result 'exit-code'.
2026-07-22T10:11:15+03:00 pve01 systemd[1]: Failed to start homelab-backup-matrix-refresh.service - Generate unified homelab backup matrix state.
========== NODE=pve02 ==========
SSH_STATUS=OK
HOST=pve02
UTC_NOW=2026-07-22T14:53:54Z
=== FAILED_UNITS ===
FAILED_UNIT_COUNT=0
=== TARGET_UNITS ===
UNIT_MISSING=homelab-backup-matrix-refresh.service
UNIT_MISSING=homelab-backup-matrix-refresh.timer
UNIT_MISSING=homelab-vm160-remote-backup.service
UNIT_MISSING=homelab-vm160-remote-backup.timer
UNIT_MISSING=homelab-cluster-admin-pull.service
UNIT_MISSING=homelab-cluster-admin-pull.timer
UNIT_MISSING=homelab-vm-local-dumps-2cloud.service
UNIT_MISSING=homelab-vm-local-dumps-2cloud.timer
UNIT_MISSING=homelab-mail-cloud-critical-upload.service
UNIT_MISSING=homelab-mail-cloud-critical-upload.timer
UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service
UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer
UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service
UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer
UNIT_MISSING=homelab-mail-cloud-restore-drill.service
UNIT_MISSING=homelab-mail-cloud-restore-drill.timer
UNIT_MISSING=homelab-verified-backup-health.service
UNIT_MISSING=homelab-verified-backup-health.timer
UNIT_MISSING=homelab-backup-sla-health.service
UNIT_MISSING=homelab-backup-sla-health.timer
=== RELEVANT_TIMERS ===
=== TARGET_SCRIPT_STATE ===
FILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh
FILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup
FILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull
FILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud
FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload
FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload
FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload
FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill
=== CURRENT_BACKUP_HEALTH ===
HEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt
=== RCLONE_REMOTE_NAMES ===
pve01-mail-01:
pve01-mail-01-crypt:
pve01-mail-02:
pve01-mail-02-crypt:
pve01-mail-03:
pve01-mail-03-crypt:
pve01-mail-04:
pve01-mail-04-crypt:
pve01-mail-05:
pve01-mail-05-crypt:
pve02-mail-01:
pve02-mail-01-crypt:
pve02-mail-02:
pve02-mail-02-crypt:
pve02-mail-03:
pve02-mail-03-crypt:
pve02-mail-04:
pve02-mail-04-crypt:
pve03-mail-01:
pve03-mail-01-crypt:
pve03-mail-02:
pve03-mail-02-crypt:
pve03-mail-03:
pve03-mail-03-crypt:
pve03-mail-04:
pve03-mail-04-crypt:
=== ACTIVE_VZDUMP ===
3930634 bash -c              set -u              echo HOST=$(hostname -s)             echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in                 homelab-backup-matrix-refresh.service                 homelab-backup-matrix-refresh.timer                 homelab-vm160-remote-backup.service                 homelab-vm160-remote-backup.timer                 homelab-cluster-admin-pull.service                 homelab-cluster-admin-pull.timer                 homelab-vm-local-dumps-2cloud.service                 homelab-vm-local-dumps-2cloud.timer                 homelab-mail-cloud-critical-upload.service                 homelab-mail-cloud-critical-upload.timer                 homelab-mail-cloud-edge-vm-upload.service                 homelab-mail-cloud-edge-vm-upload.timer                 homelab-mail-cloud-nextcloud-vm-upload.service                 homelab-mail-cloud-nextcloud-vm-upload.timer                 homelab-mail-cloud-restore-drill.service                 homelab-mail-cloud-restore-drill.timer                 homelab-verified-backup-health.service                 homelab-verified-backup-health.timer                 homelab-backup-sla-health.service                 homelab-backup-sla-health.timer             do                 if systemctl cat "$U" >/dev/null 2>&1                 then                     systemctl show "$U"                         -p Id                         -p LoadState                         -p ActiveState                         -p SubState                         -p Result                         -p ExecMainStatus                         -p UnitFileState                         -p ActiveEnterTimestamp                         -p InactiveEnterTimestamp                         -p ExecStart                         --no-pager                 else                     echo UNIT_MISSING=$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in                 /usr/local/sbin/homelab-backup-matrix-refresh                 /usr/local/sbin/homelab-vm160-remote-backup                 /usr/local/sbin/homelab-cluster-admin-pull                 /usr/local/sbin/homelab-vm-local-dumps-2cloud                 /usr/local/sbin/homelab-mail-cloud-critical-upload                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e "$F"                 then                     stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F"                      if /usr/bin/bash -n "$F" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=$F:OK                     else                         echo SCRIPT_SYNTAX=$F:FAIL                     fi                 else                     echo FILE_MISSING=$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in                 /var/lib/homelab-health/unified-backup-matrix.txt                 /var/lib/homelab-health/backup-framework.txt                 /var/lib/homelab-health/backup-sla.txt                 /var/lib/homelab-health/backup-coverage-matrix.txt                 /var/lib/homelab-health/vm-local-dumps-cloud.txt                 /var/lib/homelab-health/vm-local-dumps-retention.txt                 /var/lib/homelab-health/vm-backup.txt                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f "$H"                 then                     echo HEALTH_FILE=$H                     stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H"                     sed -n "1p" "$H"                 else                     echo HEALTH_FILE_MISSING=$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve                 -type f                 \( -name "vzdump-*.zst"                    -o -name "vzdump-*.lzo"                    -o -name "vzdump-*.gz" \)                 -printf x                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve                 -type f                 \( -name "vzdump-*.zst"                    -o -name "vzdump-*.lzo"                    -o -name "vzdump-*.gz" \)                 -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n"                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl                 -u homelab-backup-matrix-refresh.service                 -u homelab-vm160-remote-backup.service                 -u homelab-cluster-admin-pull.service                 -n 120                 --no-pager                 -o short-iso                 2>/dev/null ||                 true         
=== LOCAL_ARCHIVES ===
ARCHIVE_COUNT=10
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_19-03_47_26.tar.zst SIZE=284443608 MTIME=2026-07-19T03:47:41.8525503200
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_20-03_30_02.tar.zst SIZE=286538526 MTIME=2026-07-20T03:30:18.4386603270
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_21-03_30_04.tar.zst SIZE=288349157 MTIME=2026-07-21T03:30:21.2480085730
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-111-2026_07_22-03_30_01.tar.zst SIZE=290059742 MTIME=2026-07-22T03:30:22.8748248510
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_19-03_47_43.tar.zst SIZE=220049041 MTIME=2026-07-19T03:47:58.9256407960
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_20-03_30_18.tar.zst SIZE=220065825 MTIME=2026-07-20T03:30:30.8287264090
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_21-03_30_21.tar.zst SIZE=220080970 MTIME=2026-07-21T03:30:33.3910787180
ARCHIVE=/var/lib/vz/dump/vzdump-lxc-113-2026_07_22-03_30_23.tar.zst SIZE=220095608 MTIME=2026-07-22T03:30:40.6809207660
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_06_30-03_30_25.vma.zst SIZE=868048369 MTIME=2026-06-30T03:31:08.4903256780
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-160-2026_07_01-03_30_27.vma.zst SIZE=997503422 MTIME=2026-07-01T03:31:11.1700059310
=== TARGET_UNIT_LOGS ===
-- No entries --
========== NODE=pve03 ==========
SSH_STATUS=OK
HOST=pve03
UTC_NOW=2026-07-22T14:53:55Z
=== FAILED_UNITS ===
FAILED_UNIT_COUNT=1
homelab-immutable-offline-verify.service loaded failed failed Verify immutable offline backup image
=== TARGET_UNITS ===
UNIT_MISSING=homelab-backup-matrix-refresh.service
UNIT_MISSING=homelab-backup-matrix-refresh.timer
UNIT_MISSING=homelab-vm160-remote-backup.service
UNIT_MISSING=homelab-vm160-remote-backup.timer
UNIT_MISSING=homelab-cluster-admin-pull.service
UNIT_MISSING=homelab-cluster-admin-pull.timer
UNIT_MISSING=homelab-vm-local-dumps-2cloud.service
UNIT_MISSING=homelab-vm-local-dumps-2cloud.timer
UNIT_MISSING=homelab-mail-cloud-critical-upload.service
UNIT_MISSING=homelab-mail-cloud-critical-upload.timer
UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.service
UNIT_MISSING=homelab-mail-cloud-edge-vm-upload.timer
UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.service
UNIT_MISSING=homelab-mail-cloud-nextcloud-vm-upload.timer
UNIT_MISSING=homelab-mail-cloud-restore-drill.service
UNIT_MISSING=homelab-mail-cloud-restore-drill.timer
UNIT_MISSING=homelab-verified-backup-health.service
UNIT_MISSING=homelab-verified-backup-health.timer
UNIT_MISSING=homelab-backup-sla-health.service
UNIT_MISSING=homelab-backup-sla-health.timer
=== RELEVANT_TIMERS ===
=== TARGET_SCRIPT_STATE ===
FILE_MISSING=/usr/local/sbin/homelab-backup-matrix-refresh
FILE_MISSING=/usr/local/sbin/homelab-vm160-remote-backup
FILE_MISSING=/usr/local/sbin/homelab-cluster-admin-pull
FILE_MISSING=/usr/local/sbin/homelab-vm-local-dumps-2cloud
FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-critical-upload
FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-edge-vm-upload
FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload
FILE_MISSING=/usr/local/sbin/homelab-mail-cloud-restore-drill
=== CURRENT_BACKUP_HEALTH ===
HEALTH_FILE_MISSING=/var/lib/homelab-health/unified-backup-matrix.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-framework.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-sla.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/backup-coverage-matrix.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-cloud.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-local-dumps-retention.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/vm-backup.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/edge-vm-offhost-freshness.txt
HEALTH_FILE_MISSING=/var/lib/homelab-health/mail-cloud-restore-drill.txt
=== RCLONE_REMOTE_NAMES ===
pve01-mail-01:
pve01-mail-01-crypt:
pve01-mail-02:
pve01-mail-02-crypt:
pve01-mail-03:
pve01-mail-03-crypt:
pve01-mail-04:
pve01-mail-04-crypt:
pve01-mail-05:
pve01-mail-05-crypt:
pve02-mail-01:
pve02-mail-01-crypt:
pve02-mail-02:
pve02-mail-02-crypt:
pve02-mail-03:
pve02-mail-03-crypt:
pve02-mail-04:
pve02-mail-04-crypt:
pve03-mail-01:
pve03-mail-01-crypt:
pve03-mail-02:
pve03-mail-02-crypt:
pve03-mail-03:
pve03-mail-03-crypt:
pve03-mail-04:
pve03-mail-04-crypt:
=== ACTIVE_VZDUMP ===
2313989 bash -c              set -u              echo HOST=$(hostname -s)             echo UTC_NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)              echo === FAILED_UNITS ===             echo -n FAILED_UNIT_COUNT=             systemctl --failed --no-legend --plain 2>/dev/null | wc -l             systemctl --failed --no-legend --plain 2>/dev/null || true              echo === TARGET_UNITS ===             for U in                 homelab-backup-matrix-refresh.service                 homelab-backup-matrix-refresh.timer                 homelab-vm160-remote-backup.service                 homelab-vm160-remote-backup.timer                 homelab-cluster-admin-pull.service                 homelab-cluster-admin-pull.timer                 homelab-vm-local-dumps-2cloud.service                 homelab-vm-local-dumps-2cloud.timer                 homelab-mail-cloud-critical-upload.service                 homelab-mail-cloud-critical-upload.timer                 homelab-mail-cloud-edge-vm-upload.service                 homelab-mail-cloud-edge-vm-upload.timer                 homelab-mail-cloud-nextcloud-vm-upload.service                 homelab-mail-cloud-nextcloud-vm-upload.timer                 homelab-mail-cloud-restore-drill.service                 homelab-mail-cloud-restore-drill.timer                 homelab-verified-backup-health.service                 homelab-verified-backup-health.timer                 homelab-backup-sla-health.service                 homelab-backup-sla-health.timer             do                 if systemctl cat "$U" >/dev/null 2>&1                 then                     systemctl show "$U"                         -p Id                         -p LoadState                         -p ActiveState                         -p SubState                         -p Result                         -p ExecMainStatus                         -p UnitFileState                         -p ActiveEnterTimestamp                         -p InactiveEnterTimestamp                         -p ExecStart                         --no-pager                 else                     echo UNIT_MISSING=$U                 fi             done              echo === RELEVANT_TIMERS ===             systemctl list-timers --all --no-pager --no-legend 2>/dev/null |                 grep -E "homelab-(backup-matrix-refresh|vm160-remote-backup|cluster-admin-pull|vm-local-dumps-2cloud|mail-cloud|verified-backup|backup-sla)" ||                 true              echo === TARGET_SCRIPT_STATE ===             for F in                 /usr/local/sbin/homelab-backup-matrix-refresh                 /usr/local/sbin/homelab-vm160-remote-backup                 /usr/local/sbin/homelab-cluster-admin-pull                 /usr/local/sbin/homelab-vm-local-dumps-2cloud                 /usr/local/sbin/homelab-mail-cloud-critical-upload                 /usr/local/sbin/homelab-mail-cloud-edge-vm-upload                 /usr/local/sbin/homelab-mail-cloud-nextcloud-vm-upload                 /usr/local/sbin/homelab-mail-cloud-restore-drill             do                 if test -e "$F"                 then                     stat -c "FILE=%n TYPE=%F MODE=%a OWNER=%U:%G SIZE=%s MTIME=%y" "$F"                      if /usr/bin/bash -n "$F" 2>/dev/null                     then                         echo SCRIPT_SYNTAX=$F:OK                     else                         echo SCRIPT_SYNTAX=$F:FAIL                     fi                 else                     echo FILE_MISSING=$F                 fi             done              echo === CURRENT_BACKUP_HEALTH ===             for H in                 /var/lib/homelab-health/unified-backup-matrix.txt                 /var/lib/homelab-health/backup-framework.txt                 /var/lib/homelab-health/backup-sla.txt                 /var/lib/homelab-health/backup-coverage-matrix.txt                 /var/lib/homelab-health/vm-local-dumps-cloud.txt                 /var/lib/homelab-health/vm-local-dumps-retention.txt                 /var/lib/homelab-health/vm-backup.txt                 /var/lib/homelab-health/cluster-admin-vm-mail-cloud-backup.txt                 /var/lib/homelab-health/edge-vm-offhost-freshness.txt                 /var/lib/homelab-health/mail-cloud-restore-drill.txt             do                 if test -f "$H"                 then                     echo HEALTH_FILE=$H                     stat -c "HEALTH_MTIME=%y HEALTH_SIZE=%s" "$H"                     sed -n "1p" "$H"                 else                     echo HEALTH_FILE_MISSING=$H                 fi             done              echo === RCLONE_REMOTE_NAMES ===             if command -v rclone >/dev/null 2>&1             then                 rclone listremotes 2>/dev/null || echo RCLONE_LISTREMOTES=FAIL             else                 echo RCLONE=NOT_INSTALLED             fi              echo === ACTIVE_VZDUMP ===             pgrep -af "[v]zdump" || echo ACTIVE_VZDUMP=NO              echo === LOCAL_ARCHIVES ===             echo -n ARCHIVE_COUNT=             find /var/lib/vz/dump /mnt/pve                 -type f                 \( -name "vzdump-*.zst"                    -o -name "vzdump-*.lzo"                    -o -name "vzdump-*.gz" \)                 -printf x                 2>/dev/null |                 wc -c              find /var/lib/vz/dump /mnt/pve                 -type f                 \( -name "vzdump-*.zst"                    -o -name "vzdump-*.lzo"                    -o -name "vzdump-*.gz" \)                 -printf "ARCHIVE=%p SIZE=%s MTIME=%TY-%Tm-%TdT%TH:%TM:%TS\n"                 2>/dev/null |                 sort              echo === TARGET_UNIT_LOGS ===             journalctl                 -u homelab-backup-matrix-refresh.service                 -u homelab-vm160-remote-backup.service                 -u homelab-cluster-admin-pull.service                 -n 120                 --no-pager                 -o short-iso                 2>/dev/null ||                 true         
=== LOCAL_ARCHIVES ===
ARCHIVE_COUNT=8
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_19-03_30_03.vma.zst SIZE=11013723404 MTIME=2026-07-19T03:33:53.3463878040
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_20-03_30_00.vma.zst SIZE=11116472459 MTIME=2026-07-20T03:33:53.2839064370
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_21-03_30_06.vma.zst SIZE=11043834894 MTIME=2026-07-21T03:33:54.3725716230
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-130-2026_07_22-03_30_02.vma.zst SIZE=11072513783 MTIME=2026-07-22T03:33:58.8961071480
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_19-03_33_53.vma.zst SIZE=838951976 MTIME=2026-07-19T03:34:33.1034859540
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_20-03_33_54.vma.zst SIZE=838817925 MTIME=2026-07-20T03:34:37.1719936670
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_21-03_33_55.vma.zst SIZE=1137072350 MTIME=2026-07-21T03:34:37.6276811360
ARCHIVE=/var/lib/vz/dump/vzdump-qemu-190-2026_07_22-03_33_59.vma.zst SIZE=1177407443 MTIME=2026-07-22T03:34:44.3412293530
=== TARGET_UNIT_LOGS ===
-- No entries --
CHANGES_MADE=NO
SECRET_PRINTED=NO
PRIVATE_ADDRESS_PRINTED=NO
RESULT=OK
BACKUP_OFFHOST_CURRENT_STATE=COMPLETE

OUTPUT_END
CHAT_OUTPUT_END
